drivers/gpu/drm/i915/display/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/dma-resv.h>
  28 #include <linux/i2c.h>
  29 #include <linux/input.h>
  30 #include <linux/kernel.h>
  31 #include <linux/module.h>
  32 #include <linux/slab.h>
  33 #include <linux/string_helpers.h>
  34 #include <linux/vga_switcheroo.h>
  35 #include <acpi/video.h>
  36
  37 #include <drm/display/drm_dp_helper.h>
  38 #include <drm/drm_atomic.h>
  39 #include <drm/drm_atomic_helper.h>
  40 #include <drm/drm_atomic_uapi.h>
  41 #include <drm/drm_damage_helper.h>
  42 #include <drm/drm_edid.h>
  43 #include <drm/drm_fourcc.h>
  44 #include <drm/drm_privacy_screen_consumer.h>
  45 #include <drm/drm_probe_helper.h>
  46 #include <drm/drm_rect.h>
  47
  48 #include "gem/i915_gem_lmem.h"
  49 #include "gem/i915_gem_object.h"
  50
  51 #include "g4x_dp.h"
  52 #include "g4x_hdmi.h"
  53 #include "hsw_ips.h"
  54 #include "i915_drv.h"
  55 #include "i915_reg.h"
  56 #include "i915_utils.h"
  57 #include "i9xx_plane.h"
  58 #include "i9xx_wm.h"
  59 #include "icl_dsi.h"
  60 #include "intel_acpi.h"
  61 #include "intel_atomic.h"
  62 #include "intel_atomic_plane.h"
  63 #include "intel_audio.h"
  64 #include "intel_bw.h"
  65 #include "intel_cdclk.h"
  66 #include "intel_color.h"
  67 #include "intel_crt.h"
  68 #include "intel_crtc.h"
  69 #include "intel_crtc_state_dump.h"
  70 #include "intel_ddi.h"
  71 #include "intel_de.h"
  72 #include "intel_display_debugfs.h"
  73 #include "intel_display_power.h"
  74 #include "intel_display_types.h"
  75 #include "intel_dmc.h"
  76 #include "intel_dp.h"
  77 #include "intel_dp_link_training.h"
  78 #include "intel_dp_mst.h"
  79 #include "intel_dpio_phy.h"
  80 #include "intel_dpll.h"
  81 #include "intel_dpll_mgr.h"
  82 #include "intel_dpt.h"
  83 #include "intel_drrs.h"
  84 #include "intel_dsi.h"
  85 #include "intel_dvo.h"
  86 #include "intel_fb.h"
  87 #include "intel_fbc.h"
  88 #include "intel_fbdev.h"
  89 #include "intel_fdi.h"
  90 #include "intel_fifo_underrun.h"
  91 #include "intel_frontbuffer.h"
  92 #include "intel_gmbus.h"
  93 #include "intel_hdcp.h"
  94 #include "intel_hdmi.h"
  95 #include "intel_hotplug.h"
  96 #include "intel_hti.h"
  97 #include "intel_lvds.h"
  98 #include "intel_lvds_regs.h"
  99 #include "intel_modeset_setup.h"
 100 #include "intel_modeset_verify.h"
 101 #include "intel_overlay.h"
 102 #include "intel_panel.h"
 103 #include "intel_pch_display.h"
 104 #include "intel_pch_refclk.h"
 105 #include "intel_pcode.h"
 106 #include "intel_pipe_crc.h"
 107 #include "intel_plane_initial.h"
 108 #include "intel_pm.h"
 109 #include "intel_pps.h"
 110 #include "intel_psr.h"
 111 #include "intel_quirks.h"
 112 #include "intel_sdvo.h"
 113 #include "intel_snps_phy.h"
 114 #include "intel_tc.h"
 115 #include "intel_tv.h"
 116 #include "intel_vblank.h"
 117 #include "intel_vdsc.h"
 118 #include "intel_vdsc_regs.h"
 119 #include "intel_vga.h"
 120 #include "intel_vrr.h"
 121 #include "intel_wm.h"
 122 #include "skl_scaler.h"
 123 #include "skl_universal_plane.h"
 124 #include "skl_watermark.h"
 125 #include "vlv_dsi.h"
 126 #include "vlv_dsi_pll.h"
 127 #include "vlv_dsi_regs.h"
 128 #include "vlv_sideband.h"
 129
 130 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state);
 131 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state);
 132 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state);
 133 static void bdw_set_pipe_misc(const struct intel_crtc_state *crtc_state);
 134 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
 135
 136 /* returns HPLL frequency in kHz */
 137 int vlv_get_hpll_vco(struct drm_i915_private *dev_priv)
 138 {
 139         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 140
 141         /* Obtain SKU information */
 142         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
 143                 CCK_FUSE_HPLL_FREQ_MASK;
 144
 145         return vco_freq[hpll_freq] * 1000;
 146 }
 147
 148 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
 149                       const char *name, u32 reg, int ref_freq)
 150 {
 151         u32 val;
 152         int divider;
 153
 154         val = vlv_cck_read(dev_priv, reg);
 155         divider = val & CCK_FREQUENCY_VALUES;
 156
 157         drm_WARN(&dev_priv->drm, (val & CCK_FREQUENCY_STATUS) !=
 158                  (divider << CCK_FREQUENCY_STATUS_SHIFT),
 159                  "%s change in progress\n", name);
 160
 161         return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1);
 162 }
 163
 164 int vlv_get_cck_clock_hpll(struct drm_i915_private *dev_priv,
 165                            const char *name, u32 reg)
 166 {
 167         int hpll;
 168
 169         vlv_cck_get(dev_priv);
 170
 171         if (dev_priv->hpll_freq == 0)
 172                 dev_priv->hpll_freq = vlv_get_hpll_vco(dev_priv);
 173
 174         hpll = vlv_get_cck_clock(dev_priv, name, reg, dev_priv->hpll_freq);
 175
 176         vlv_cck_put(dev_priv);
 177
 178         return hpll;
 179 }
 180
 181 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 182 {
 183         if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
 184                 return;
 185
 186         dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
 187                                                       CCK_CZ_CLOCK_CONTROL);
 188
 189         drm_dbg(&dev_priv->drm, "CZ clock rate: %d kHz\n",
 190                 dev_priv->czclk_freq);
 191 }
 192
 193 static bool is_hdr_mode(const struct intel_crtc_state *crtc_state)
 194 {
 195         return (crtc_state->active_planes &
 196                 ~(icl_hdr_plane_mask() | BIT(PLANE_CURSOR))) == 0;
 197 }
 198
 199 /* WA Display #0827: Gen9:all */
 200 static void
 201 skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable)
 202 {
 203         if (enable)
 204                 intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe),
 205                              0, DUPS1_GATING_DIS | DUPS2_GATING_DIS);
 206         else
 207                 intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe),
 208                              DUPS1_GATING_DIS | DUPS2_GATING_DIS, 0);
 209 }
 210
 211 /* Wa_2006604312:icl,ehl */
 212 static void
 213 icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 214                        bool enable)
 215 {
 216         if (enable)
 217                 intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe), 0, DPFR_GATING_DIS);
 218         else
 219                 intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe), DPFR_GATING_DIS, 0);
 220 }
 221
 222 /* Wa_1604331009:icl,jsl,ehl */
 223 static void
 224 icl_wa_cursorclkgating(struct drm_i915_private *dev_priv, enum pipe pipe,
 225                        bool enable)
 226 {
 227         intel_de_rmw(dev_priv, CLKGATE_DIS_PSL(pipe), CURSOR_GATING_DIS,
 228                      enable ? CURSOR_GATING_DIS : 0);
 229 }
 230
 231 static bool
 232 is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state)
 233 {
 234         return crtc_state->master_transcoder != INVALID_TRANSCODER;
 235 }
 236
 237 static bool
 238 is_trans_port_sync_master(const struct intel_crtc_state *crtc_state)
 239 {
 240         return crtc_state->sync_mode_slaves_mask != 0;
 241 }
 242
 243 bool
 244 is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state)
 245 {
 246         return is_trans_port_sync_master(crtc_state) ||
 247                 is_trans_port_sync_slave(crtc_state);
 248 }
 249
 250 static enum pipe bigjoiner_master_pipe(const struct intel_crtc_state *crtc_state)
 251 {
 252         return ffs(crtc_state->bigjoiner_pipes) - 1;
 253 }
 254
 255 u8 intel_crtc_bigjoiner_slave_pipes(const struct intel_crtc_state *crtc_state)
 256 {
 257         if (crtc_state->bigjoiner_pipes)
 258                 return crtc_state->bigjoiner_pipes & ~BIT(bigjoiner_master_pipe(crtc_state));
 259         else
 260                 return 0;
 261 }
 262
 263 bool intel_crtc_is_bigjoiner_slave(const struct intel_crtc_state *crtc_state)
 264 {
 265         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 266
 267         return crtc_state->bigjoiner_pipes &&
 268                 crtc->pipe != bigjoiner_master_pipe(crtc_state);
 269 }
 270
 271 bool intel_crtc_is_bigjoiner_master(const struct intel_crtc_state *crtc_state)
 272 {
 273         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 274
 275         return crtc_state->bigjoiner_pipes &&
 276                 crtc->pipe == bigjoiner_master_pipe(crtc_state);
 277 }
 278
 279 static int intel_bigjoiner_num_pipes(const struct intel_crtc_state *crtc_state)
 280 {
 281         return hweight8(crtc_state->bigjoiner_pipes);
 282 }
 283
 284 struct intel_crtc *intel_master_crtc(const struct intel_crtc_state *crtc_state)
 285 {
 286         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 287
 288         if (intel_crtc_is_bigjoiner_slave(crtc_state))
 289                 return intel_crtc_for_pipe(i915, bigjoiner_master_pipe(crtc_state));
 290         else
 291                 return to_intel_crtc(crtc_state->uapi.crtc);
 292 }
 293
 294 static void
 295 intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state)
 296 {
 297         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 298         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 299
 300         if (DISPLAY_VER(dev_priv) >= 4) {
 301                 enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 302
 303                 /* Wait for the Pipe State to go off */
 304                 if (intel_de_wait_for_clear(dev_priv, TRANSCONF(cpu_transcoder),
 305                                             TRANSCONF_STATE_ENABLE, 100))
 306                         drm_WARN(&dev_priv->drm, 1, "pipe_off wait timed out\n");
 307         } else {
 308                 intel_wait_for_pipe_scanline_stopped(crtc);
 309         }
 310 }
 311
 312 void assert_transcoder(struct drm_i915_private *dev_priv,
 313                        enum transcoder cpu_transcoder, bool state)
 314 {
 315         bool cur_state;
 316         enum intel_display_power_domain power_domain;
 317         intel_wakeref_t wakeref;
 318
 319         /* we keep both pipes enabled on 830 */
 320         if (IS_I830(dev_priv))
 321                 state = true;
 322
 323         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
 324         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
 325         if (wakeref) {
 326                 u32 val = intel_de_read(dev_priv, TRANSCONF(cpu_transcoder));
 327                 cur_state = !!(val & TRANSCONF_ENABLE);
 328
 329                 intel_display_power_put(dev_priv, power_domain, wakeref);
 330         } else {
 331                 cur_state = false;
 332         }
 333
 334         I915_STATE_WARN(cur_state != state,
 335                         "transcoder %s assertion failure (expected %s, current %s)\n",
 336                         transcoder_name(cpu_transcoder),
 337                         str_on_off(state), str_on_off(cur_state));
 338 }
 339
 340 static void assert_plane(struct intel_plane *plane, bool state)
 341 {
 342         enum pipe pipe;
 343         bool cur_state;
 344
 345         cur_state = plane->get_hw_state(plane, &pipe);
 346
 347         I915_STATE_WARN(cur_state != state,
 348                         "%s assertion failure (expected %s, current %s)\n",
 349                         plane->base.name, str_on_off(state),
 350                         str_on_off(cur_state));
 351 }
 352
 353 #define assert_plane_enabled(p) assert_plane(p, true)
 354 #define assert_plane_disabled(p) assert_plane(p, false)
 355
 356 static void assert_planes_disabled(struct intel_crtc *crtc)
 357 {
 358         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 359         struct intel_plane *plane;
 360
 361         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
 362                 assert_plane_disabled(plane);
 363 }
 364
 365 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 366                          struct intel_digital_port *dig_port,
 367                          unsigned int expected_mask)
 368 {
 369         u32 port_mask;
 370         i915_reg_t dpll_reg;
 371
 372         switch (dig_port->base.port) {
 373         default:
 374                 MISSING_CASE(dig_port->base.port);
 375                 fallthrough;
 376         case PORT_B:
 377                 port_mask = DPLL_PORTB_READY_MASK;
 378                 dpll_reg = DPLL(0);
 379                 break;
 380         case PORT_C:
 381                 port_mask = DPLL_PORTC_READY_MASK;
 382                 dpll_reg = DPLL(0);
 383                 expected_mask <<= 4;
 384                 break;
 385         case PORT_D:
 386                 port_mask = DPLL_PORTD_READY_MASK;
 387                 dpll_reg = DPIO_PHY_STATUS;
 388                 break;
 389         }
 390
 391         if (intel_de_wait_for_register(dev_priv, dpll_reg,
 392                                        port_mask, expected_mask, 1000))
 393                 drm_WARN(&dev_priv->drm, 1,
 394                          "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n",
 395                          dig_port->base.base.base.id, dig_port->base.base.name,
 396                          intel_de_read(dev_priv, dpll_reg) & port_mask,
 397                          expected_mask);
 398 }
 399
 400 void intel_enable_transcoder(const struct intel_crtc_state *new_crtc_state)
 401 {
 402         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
 403         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 404         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
 405         enum pipe pipe = crtc->pipe;
 406         i915_reg_t reg;
 407         u32 val;
 408
 409         drm_dbg_kms(&dev_priv->drm, "enabling pipe %c\n", pipe_name(pipe));
 410
 411         assert_planes_disabled(crtc);
 412
 413         /*
 414          * A pipe without a PLL won't actually be able to drive bits from
 415          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
 416          * need the check.
 417          */
 418         if (HAS_GMCH(dev_priv)) {
 419                 if (intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
 420                         assert_dsi_pll_enabled(dev_priv);
 421                 else
 422                         assert_pll_enabled(dev_priv, pipe);
 423         } else {
 424                 if (new_crtc_state->has_pch_encoder) {
 425                         /* if driving the PCH, we need FDI enabled */
 426                         assert_fdi_rx_pll_enabled(dev_priv,
 427                                                   intel_crtc_pch_transcoder(crtc));
 428                         assert_fdi_tx_pll_enabled(dev_priv,
 429                                                   (enum pipe) cpu_transcoder);
 430                 }
 431                 /* FIXME: assert CPU port conditions for SNB+ */
 432         }
 433
 434         /* Wa_22012358565:adl-p */
 435         if (DISPLAY_VER(dev_priv) == 13)
 436                 intel_de_rmw(dev_priv, PIPE_ARB_CTL(pipe),
 437                              0, PIPE_ARB_USE_PROG_SLOTS);
 438
 439         reg = TRANSCONF(cpu_transcoder);
 440         val = intel_de_read(dev_priv, reg);
 441         if (val & TRANSCONF_ENABLE) {
 442                 /* we keep both pipes enabled on 830 */
 443                 drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv));
 444                 return;
 445         }
 446
 447         intel_de_write(dev_priv, reg, val | TRANSCONF_ENABLE);
 448         intel_de_posting_read(dev_priv, reg);
 449
 450         /*
 451          * Until the pipe starts PIPEDSL reads will return a stale value,
 452          * which causes an apparent vblank timestamp jump when PIPEDSL
 453          * resets to its proper value. That also messes up the frame count
 454          * when it's derived from the timestamps. So let's wait for the
 455          * pipe to start properly before we call drm_crtc_vblank_on()
 456          */
 457         if (intel_crtc_max_vblank_count(new_crtc_state) == 0)
 458                 intel_wait_for_pipe_scanline_moving(crtc);
 459 }
 460
 461 void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 462 {
 463         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 464         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 465         enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 466         enum pipe pipe = crtc->pipe;
 467         i915_reg_t reg;
 468         u32 val;
 469
 470         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c\n", pipe_name(pipe));
 471
 472         /*
 473          * Make sure planes won't keep trying to pump pixels to us,
 474          * or we might hang the display.
 475          */
 476         assert_planes_disabled(crtc);
 477
 478         reg = TRANSCONF(cpu_transcoder);
 479         val = intel_de_read(dev_priv, reg);
 480         if ((val & TRANSCONF_ENABLE) == 0)
 481                 return;
 482
 483         /*
 484          * Double wide has implications for planes
 485          * so best keep it disabled when not needed.
 486          */
 487         if (old_crtc_state->double_wide)
 488                 val &= ~TRANSCONF_DOUBLE_WIDE;
 489
 490         /* Don't disable pipe or pipe PLLs if needed */
 491         if (!IS_I830(dev_priv))
 492                 val &= ~TRANSCONF_ENABLE;
 493
 494         if (DISPLAY_VER(dev_priv) >= 14)
 495                 intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(cpu_transcoder),
 496                              FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
 497         else if (DISPLAY_VER(dev_priv) >= 12)
 498                 intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder),
 499                              FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
 500
 501         intel_de_write(dev_priv, reg, val);
 502         if ((val & TRANSCONF_ENABLE) == 0)
 503                 intel_wait_for_pipe_off(old_crtc_state);
 504 }
 505
 506 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info)
 507 {
 508         unsigned int size = 0;
 509         int i;
 510
 511         for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++)
 512                 size += rot_info->plane[i].dst_stride * rot_info->plane[i].width;
 513
 514         return size;
 515 }
 516
 517 unsigned int intel_remapped_info_size(const struct intel_remapped_info *rem_info)
 518 {
 519         unsigned int size = 0;
 520         int i;
 521
 522         for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
 523                 unsigned int plane_size;
 524
 525                 if (rem_info->plane[i].linear)
 526                         plane_size = rem_info->plane[i].size;
 527                 else
 528                         plane_size = rem_info->plane[i].dst_stride * rem_info->plane[i].height;
 529
 530                 if (plane_size == 0)
 531                         continue;
 532
 533                 if (rem_info->plane_alignment)
 534                         size = ALIGN(size, rem_info->plane_alignment);
 535
 536                 size += plane_size;
 537         }
 538
 539         return size;
 540 }
 541
 542 bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
 543 {
 544         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 545         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 546
 547         return DISPLAY_VER(dev_priv) < 4 ||
 548                 (plane->fbc &&
 549                  plane_state->view.gtt.type == I915_GTT_VIEW_NORMAL);
 550 }
 551
 552 /*
 553  * Convert the x/y offsets into a linear offset.
 554  * Only valid with 0/180 degree rotation, which is fine since linear
 555  * offset is only used with linear buffers on pre-hsw and tiled buffers
 556  * with gen2/3, and 90/270 degree rotations isn't supported on any of them.
 557  */
 558 u32 intel_fb_xy_to_linear(int x, int y,
 559                           const struct intel_plane_state *state,
 560                           int color_plane)
 561 {
 562         const struct drm_framebuffer *fb = state->hw.fb;
 563         unsigned int cpp = fb->format->cpp[color_plane];
 564         unsigned int pitch = state->view.color_plane[color_plane].mapping_stride;
 565
 566         return y * pitch + x * cpp;
 567 }
 568
 569 /*
 570  * Add the x/y offsets derived from fb->offsets[] to the user
 571  * specified plane src x/y offsets. The resulting x/y offsets
 572  * specify the start of scanout from the beginning of the gtt mapping.
 573  */
 574 void intel_add_fb_offsets(int *x, int *y,
 575                           const struct intel_plane_state *state,
 576                           int color_plane)
 577
 578 {
 579         *x += state->view.color_plane[color_plane].x;
 580         *y += state->view.color_plane[color_plane].y;
 581 }
 582
 583 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
 584                               u32 pixel_format, u64 modifier)
 585 {
 586         struct intel_crtc *crtc;
 587         struct intel_plane *plane;
 588
 589         if (!HAS_DISPLAY(dev_priv))
 590                 return 0;
 591
 592         /*
 593          * We assume the primary plane for pipe A has
 594          * the highest stride limits of them all,
 595          * if in case pipe A is disabled, use the first pipe from pipe_mask.
 596          */
 597         crtc = intel_first_crtc(dev_priv);
 598         if (!crtc)
 599                 return 0;
 600
 601         plane = to_intel_plane(crtc->base.primary);
 602
 603         return plane->max_stride(plane, pixel_format, modifier,
 604                                  DRM_MODE_ROTATE_0);
 605 }
 606
 607 void intel_set_plane_visible(struct intel_crtc_state *crtc_state,
 608                              struct intel_plane_state *plane_state,
 609                              bool visible)
 610 {
 611         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 612
 613         plane_state->uapi.visible = visible;
 614
 615         if (visible)
 616                 crtc_state->uapi.plane_mask |= drm_plane_mask(&plane->base);
 617         else
 618                 crtc_state->uapi.plane_mask &= ~drm_plane_mask(&plane->base);
 619 }
 620
 621 void intel_plane_fixup_bitmasks(struct intel_crtc_state *crtc_state)
 622 {
 623         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 624         struct drm_plane *plane;
 625
 626         /*
 627          * Active_planes aliases if multiple "primary" or cursor planes
 628          * have been used on the same (or wrong) pipe. plane_mask uses
 629          * unique ids, hence we can use that to reconstruct active_planes.
 630          */
 631         crtc_state->enabled_planes = 0;
 632         crtc_state->active_planes = 0;
 633
 634         drm_for_each_plane_mask(plane, &dev_priv->drm,
 635                                 crtc_state->uapi.plane_mask) {
 636                 crtc_state->enabled_planes |= BIT(to_intel_plane(plane)->id);
 637                 crtc_state->active_planes |= BIT(to_intel_plane(plane)->id);
 638         }
 639 }
 640
 641 void intel_plane_disable_noatomic(struct intel_crtc *crtc,
 642                                   struct intel_plane *plane)
 643 {
 644         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 645         struct intel_crtc_state *crtc_state =
 646                 to_intel_crtc_state(crtc->base.state);
 647         struct intel_plane_state *plane_state =
 648                 to_intel_plane_state(plane->base.state);
 649
 650         drm_dbg_kms(&dev_priv->drm,
 651                     "Disabling [PLANE:%d:%s] on [CRTC:%d:%s]\n",
 652                     plane->base.base.id, plane->base.name,
 653                     crtc->base.base.id, crtc->base.name);
 654
 655         intel_set_plane_visible(crtc_state, plane_state, false);
 656         intel_plane_fixup_bitmasks(crtc_state);
 657         crtc_state->data_rate[plane->id] = 0;
 658         crtc_state->data_rate_y[plane->id] = 0;
 659         crtc_state->rel_data_rate[plane->id] = 0;
 660         crtc_state->rel_data_rate_y[plane->id] = 0;
 661         crtc_state->min_cdclk[plane->id] = 0;
 662
 663         if ((crtc_state->active_planes & ~BIT(PLANE_CURSOR)) == 0 &&
 664             hsw_ips_disable(crtc_state)) {
 665                 crtc_state->ips_enabled = false;
 666                 intel_crtc_wait_for_next_vblank(crtc);
 667         }
 668
 669         /*
 670          * Vblank time updates from the shadow to live plane control register
 671          * are blocked if the memory self-refresh mode is active at that
 672          * moment. So to make sure the plane gets truly disabled, disable
 673          * first the self-refresh mode. The self-refresh enable bit in turn
 674          * will be checked/applied by the HW only at the next frame start
 675          * event which is after the vblank start event, so we need to have a
 676          * wait-for-vblank between disabling the plane and the pipe.
 677          */
 678         if (HAS_GMCH(dev_priv) &&
 679             intel_set_memory_cxsr(dev_priv, false))
 680                 intel_crtc_wait_for_next_vblank(crtc);
 681
 682         /*
 683          * Gen2 reports pipe underruns whenever all planes are disabled.
 684          * So disable underrun reporting before all the planes get disabled.
 685          */
 686         if (DISPLAY_VER(dev_priv) == 2 && !crtc_state->active_planes)
 687                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false);
 688
 689         intel_plane_disable_arm(plane, crtc_state);
 690         intel_crtc_wait_for_next_vblank(crtc);
 691 }
 692
 693 unsigned int
 694 intel_plane_fence_y_offset(const struct intel_plane_state *plane_state)
 695 {
 696         int x = 0, y = 0;
 697
 698         intel_plane_adjust_aligned_offset(&x, &y, plane_state, 0,
 699                                           plane_state->view.color_plane[0].offset, 0);
 700
 701         return y;
 702 }
 703
 704 static int
 705 intel_display_commit_duplicated_state(struct intel_atomic_state *state,
 706                                       struct drm_modeset_acquire_ctx *ctx)
 707 {
 708         struct drm_i915_private *i915 = to_i915(state->base.dev);
 709         int ret;
 710
 711         ret = drm_atomic_helper_commit_duplicated_state(&state->base, ctx);
 712
 713         drm_WARN_ON(&i915->drm, ret == -EDEADLK);
 714
 715         return ret;
 716 }
 717
 718 static int
 719 __intel_display_resume(struct drm_i915_private *i915,
 720                        struct drm_atomic_state *state,
 721                        struct drm_modeset_acquire_ctx *ctx)
 722 {
 723         struct drm_crtc_state *crtc_state;
 724         struct drm_crtc *crtc;
 725         int i;
 726
 727         intel_modeset_setup_hw_state(i915, ctx);
 728         intel_vga_redisable(i915);
 729
 730         if (!state)
 731                 return 0;
 732
 733         /*
 734          * We've duplicated the state, pointers to the old state are invalid.
 735          *
 736          * Don't attempt to use the old state until we commit the duplicated state.
 737          */
 738         for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 739                 /*
 740                  * Force recalculation even if we restore
 741                  * current state. With fast modeset this may not result
 742                  * in a modeset when the state is compatible.
 743                  */
 744                 crtc_state->mode_changed = true;
 745         }
 746
 747         /* ignore any reset values/BIOS leftovers in the WM registers */
 748         if (!HAS_GMCH(i915))
 749                 to_intel_atomic_state(state)->skip_intermediate_wm = true;
 750
 751         return intel_display_commit_duplicated_state(to_intel_atomic_state(state), ctx);
 752 }
 753
 754 static bool gpu_reset_clobbers_display(struct drm_i915_private *dev_priv)
 755 {
 756         return (INTEL_INFO(dev_priv)->gpu_reset_clobbers_display &&
 757                 intel_has_gpu_reset(to_gt(dev_priv)));
 758 }
 759
 760 void intel_display_prepare_reset(struct drm_i915_private *dev_priv)
 761 {
 762         struct drm_modeset_acquire_ctx *ctx = &dev_priv->display.restore.reset_ctx;
 763         struct drm_atomic_state *state;
 764         int ret;
 765
 766         if (!HAS_DISPLAY(dev_priv))
 767                 return;
 768
 769         /* reset doesn't touch the display */
 770         if (!dev_priv->params.force_reset_modeset_test &&
 771             !gpu_reset_clobbers_display(dev_priv))
 772                 return;
 773
 774         /* We have a modeset vs reset deadlock, defensively unbreak it. */
 775         set_bit(I915_RESET_MODESET, &to_gt(dev_priv)->reset.flags);
 776         smp_mb__after_atomic();
 777         wake_up_bit(&to_gt(dev_priv)->reset.flags, I915_RESET_MODESET);
 778
 779         if (atomic_read(&dev_priv->gpu_error.pending_fb_pin)) {
 780                 drm_dbg_kms(&dev_priv->drm,
 781                             "Modeset potentially stuck, unbreaking through wedging\n");
 782                 intel_gt_set_wedged(to_gt(dev_priv));
 783         }
 784
 785         /*
 786          * Need mode_config.mutex so that we don't
 787          * trample ongoing ->detect() and whatnot.
 788          */
 789         mutex_lock(&dev_priv->drm.mode_config.mutex);
 790         drm_modeset_acquire_init(ctx, 0);
 791         while (1) {
 792                 ret = drm_modeset_lock_all_ctx(&dev_priv->drm, ctx);
 793                 if (ret != -EDEADLK)
 794                         break;
 795
 796                 drm_modeset_backoff(ctx);
 797         }
 798         /*
 799          * Disabling the crtcs gracefully seems nicer. Also the
 800          * g33 docs say we should at least disable all the planes.
 801          */
 802         state = drm_atomic_helper_duplicate_state(&dev_priv->drm, ctx);
 803         if (IS_ERR(state)) {
 804                 ret = PTR_ERR(state);
 805                 drm_err(&dev_priv->drm, "Duplicating state failed with %i\n",
 806                         ret);
 807                 return;
 808         }
 809
 810         ret = drm_atomic_helper_disable_all(&dev_priv->drm, ctx);
 811         if (ret) {
 812                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
 813                         ret);
 814                 drm_atomic_state_put(state);
 815                 return;
 816         }
 817
 818         dev_priv->display.restore.modeset_state = state;
 819         state->acquire_ctx = ctx;
 820 }
 821
 822 void intel_display_finish_reset(struct drm_i915_private *i915)
 823 {
 824         struct drm_modeset_acquire_ctx *ctx = &i915->display.restore.reset_ctx;
 825         struct drm_atomic_state *state;
 826         int ret;
 827
 828         if (!HAS_DISPLAY(i915))
 829                 return;
 830
 831         /* reset doesn't touch the display */
 832         if (!test_bit(I915_RESET_MODESET, &to_gt(i915)->reset.flags))
 833                 return;
 834
 835         state = fetch_and_zero(&i915->display.restore.modeset_state);
 836         if (!state)
 837                 goto unlock;
 838
 839         /* reset doesn't touch the display */
 840         if (!gpu_reset_clobbers_display(i915)) {
 841                 /* for testing only restore the display */
 842                 ret = intel_display_commit_duplicated_state(to_intel_atomic_state(state), ctx);
 843                 if (ret)
 844                         drm_err(&i915->drm,
 845                                 "Restoring old state failed with %i\n", ret);
 846         } else {
 847                 /*
 848                  * The display has been reset as well,
 849                  * so need a full re-initialization.
 850                  */
 851                 intel_pps_unlock_regs_wa(i915);
 852                 intel_modeset_init_hw(i915);
 853                 intel_init_clock_gating(i915);
 854                 intel_hpd_init(i915);
 855
 856                 ret = __intel_display_resume(i915, state, ctx);
 857                 if (ret)
 858                         drm_err(&i915->drm,
 859                                 "Restoring old state failed with %i\n", ret);
 860
 861                 intel_hpd_poll_disable(i915);
 862         }
 863
 864         drm_atomic_state_put(state);
 865 unlock:
 866         drm_modeset_drop_locks(ctx);
 867         drm_modeset_acquire_fini(ctx);
 868         mutex_unlock(&i915->drm.mode_config.mutex);
 869
 870         clear_bit_unlock(I915_RESET_MODESET, &to_gt(i915)->reset.flags);
 871 }
 872
 873 static void icl_set_pipe_chicken(const struct intel_crtc_state *crtc_state)
 874 {
 875         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 876         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 877         enum pipe pipe = crtc->pipe;
 878         u32 tmp;
 879
 880         tmp = intel_de_read(dev_priv, PIPE_CHICKEN(pipe));
 881
 882         /*
 883          * Display WA #1153: icl
 884          * enable hardware to bypass the alpha math
 885          * and rounding for per-pixel values 00 and 0xff
 886          */
 887         tmp |= PER_PIXEL_ALPHA_BYPASS_EN;
 888         /*
 889          * Display WA # 1605353570: icl
 890          * Set the pixel rounding bit to 1 for allowing
 891          * passthrough of Frame buffer pixels unmodified
 892          * across pipe
 893          */
 894         tmp |= PIXEL_ROUNDING_TRUNC_FB_PASSTHRU;
 895
 896         /*
 897          * Underrun recovery must always be disabled on display 13+.
 898          * DG2 chicken bit meaning is inverted compared to other platforms.
 899          */
 900         if (IS_DG2(dev_priv))
 901                 tmp &= ~UNDERRUN_RECOVERY_ENABLE_DG2;
 902         else if (DISPLAY_VER(dev_priv) >= 13)
 903                 tmp |= UNDERRUN_RECOVERY_DISABLE_ADLP;
 904
 905         /* Wa_14010547955:dg2 */
 906         if (IS_DG2_DISPLAY_STEP(dev_priv, STEP_B0, STEP_FOREVER))
 907                 tmp |= DG2_RENDER_CCSTAG_4_3_EN;
 908
 909         intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp);
 910 }
 911
 912 bool intel_has_pending_fb_unpin(struct drm_i915_private *dev_priv)
 913 {
 914         struct drm_crtc *crtc;
 915         bool cleanup_done;
 916
 917         drm_for_each_crtc(crtc, &dev_priv->drm) {
 918                 struct drm_crtc_commit *commit;
 919                 spin_lock(&crtc->commit_lock);
 920                 commit = list_first_entry_or_null(&crtc->commit_list,
 921                                                   struct drm_crtc_commit, commit_entry);
 922                 cleanup_done = commit ?
 923                         try_wait_for_completion(&commit->cleanup_done) : true;
 924                 spin_unlock(&crtc->commit_lock);
 925
 926                 if (cleanup_done)
 927                         continue;
 928
 929                 intel_crtc_wait_for_next_vblank(to_intel_crtc(crtc));
 930
 931                 return true;
 932         }
 933
 934         return false;
 935 }
 936
 937 /*
 938  * Finds the encoder associated with the given CRTC. This can only be
 939  * used when we know that the CRTC isn't feeding multiple encoders!
 940  */
 941 struct intel_encoder *
 942 intel_get_crtc_new_encoder(const struct intel_atomic_state *state,
 943                            const struct intel_crtc_state *crtc_state)
 944 {
 945         const struct drm_connector_state *connector_state;
 946         const struct drm_connector *connector;
 947         struct intel_encoder *encoder = NULL;
 948         struct intel_crtc *master_crtc;
 949         int num_encoders = 0;
 950         int i;
 951
 952         master_crtc = intel_master_crtc(crtc_state);
 953
 954         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
 955                 if (connector_state->crtc != &master_crtc->base)
 956                         continue;
 957
 958                 encoder = to_intel_encoder(connector_state->best_encoder);
 959                 num_encoders++;
 960         }
 961
 962         drm_WARN(encoder->base.dev, num_encoders != 1,
 963                  "%d encoders for pipe %c\n",
 964                  num_encoders, pipe_name(master_crtc->pipe));
 965
 966         return encoder;
 967 }
 968
 969 static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state)
 970 {
 971         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 972         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 973         const struct drm_rect *dst = &crtc_state->pch_pfit.dst;
 974         enum pipe pipe = crtc->pipe;
 975         int width = drm_rect_width(dst);
 976         int height = drm_rect_height(dst);
 977         int x = dst->x1;
 978         int y = dst->y1;
 979
 980         if (!crtc_state->pch_pfit.enabled)
 981                 return;
 982
 983         /* Force use of hard-coded filter coefficients
 984          * as some pre-programmed values are broken,
 985          * e.g. x201.
 986          */
 987         if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv))
 988                 intel_de_write_fw(dev_priv, PF_CTL(pipe), PF_ENABLE |
 989                                   PF_FILTER_MED_3x3 | PF_PIPE_SEL_IVB(pipe));
 990         else
 991                 intel_de_write_fw(dev_priv, PF_CTL(pipe), PF_ENABLE |
 992                                   PF_FILTER_MED_3x3);
 993         intel_de_write_fw(dev_priv, PF_WIN_POS(pipe), x << 16 | y);
 994         intel_de_write_fw(dev_priv, PF_WIN_SZ(pipe), width << 16 | height);
 995 }
 996
 997 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *crtc)
 998 {
 999         if (crtc->overlay)
1000                 (void) intel_overlay_switch_off(crtc->overlay);
1001
1002         /* Let userspace switch the overlay on again. In most cases userspace
1003          * has to recompute where to put it anyway.
1004          */
1005 }
1006
1007 static bool needs_nv12_wa(const struct intel_crtc_state *crtc_state)
1008 {
1009         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1010
1011         if (!crtc_state->nv12_planes)
1012                 return false;
1013
1014         /* WA Display #0827: Gen9:all */
1015         if (DISPLAY_VER(dev_priv) == 9)
1016                 return true;
1017
1018         return false;
1019 }
1020
1021 static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state)
1022 {
1023         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1024
1025         /* Wa_2006604312:icl,ehl */
1026         if (crtc_state->scaler_state.scaler_users > 0 && DISPLAY_VER(dev_priv) == 11)
1027                 return true;
1028
1029         return false;
1030 }
1031
1032 static bool needs_cursorclk_wa(const struct intel_crtc_state *crtc_state)
1033 {
1034         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1035
1036         /* Wa_1604331009:icl,jsl,ehl */
1037         if (is_hdr_mode(crtc_state) &&
1038             crtc_state->active_planes & BIT(PLANE_CURSOR) &&
1039             DISPLAY_VER(dev_priv) == 11)
1040                 return true;
1041
1042         return false;
1043 }
1044
1045 static void intel_async_flip_vtd_wa(struct drm_i915_private *i915,
1046                                     enum pipe pipe, bool enable)
1047 {
1048         if (DISPLAY_VER(i915) == 9) {
1049                 /*
1050                  * "Plane N strech max must be programmed to 11b (x1)
1051                  *  when Async flips are enabled on that plane."
1052                  */
1053                 intel_de_rmw(i915, CHICKEN_PIPESL_1(pipe),
1054                              SKL_PLANE1_STRETCH_MAX_MASK,
1055                              enable ? SKL_PLANE1_STRETCH_MAX_X1 : SKL_PLANE1_STRETCH_MAX_X8);
1056         } else {
1057                 /* Also needed on HSW/BDW albeit undocumented */
1058                 intel_de_rmw(i915, CHICKEN_PIPESL_1(pipe),
1059                              HSW_PRI_STRETCH_MAX_MASK,
1060                              enable ? HSW_PRI_STRETCH_MAX_X1 : HSW_PRI_STRETCH_MAX_X8);
1061         }
1062 }
1063
1064 static bool needs_async_flip_vtd_wa(const struct intel_crtc_state *crtc_state)
1065 {
1066         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1067
1068         return crtc_state->uapi.async_flip && i915_vtd_active(i915) &&
1069                 (DISPLAY_VER(i915) == 9 || IS_BROADWELL(i915) || IS_HASWELL(i915));
1070 }
1071
1072 static bool planes_enabling(const struct intel_crtc_state *old_crtc_state,
1073                             const struct intel_crtc_state *new_crtc_state)
1074 {
1075         return (!old_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state)) &&
1076                 new_crtc_state->active_planes;
1077 }
1078
1079 static bool planes_disabling(const struct intel_crtc_state *old_crtc_state,
1080                              const struct intel_crtc_state *new_crtc_state)
1081 {
1082         return old_crtc_state->active_planes &&
1083                 (!new_crtc_state->active_planes || intel_crtc_needs_modeset(new_crtc_state));
1084 }
1085
1086 static void intel_post_plane_update(struct intel_atomic_state *state,
1087                                     struct intel_crtc *crtc)
1088 {
1089         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1090         const struct intel_crtc_state *old_crtc_state =
1091                 intel_atomic_get_old_crtc_state(state, crtc);
1092         const struct intel_crtc_state *new_crtc_state =
1093                 intel_atomic_get_new_crtc_state(state, crtc);
1094         enum pipe pipe = crtc->pipe;
1095
1096         intel_frontbuffer_flip(dev_priv, new_crtc_state->fb_bits);
1097
1098         if (new_crtc_state->update_wm_post && new_crtc_state->hw.active)
1099                 intel_update_watermarks(dev_priv);
1100
1101         intel_fbc_post_update(state, crtc);
1102
1103         if (needs_async_flip_vtd_wa(old_crtc_state) &&
1104             !needs_async_flip_vtd_wa(new_crtc_state))
1105                 intel_async_flip_vtd_wa(dev_priv, pipe, false);
1106
1107         if (needs_nv12_wa(old_crtc_state) &&
1108             !needs_nv12_wa(new_crtc_state))
1109                 skl_wa_827(dev_priv, pipe, false);
1110
1111         if (needs_scalerclk_wa(old_crtc_state) &&
1112             !needs_scalerclk_wa(new_crtc_state))
1113                 icl_wa_scalerclkgating(dev_priv, pipe, false);
1114
1115         if (needs_cursorclk_wa(old_crtc_state) &&
1116             !needs_cursorclk_wa(new_crtc_state))
1117                 icl_wa_cursorclkgating(dev_priv, pipe, false);
1118
1119         if (intel_crtc_needs_color_update(new_crtc_state))
1120                 intel_color_post_update(new_crtc_state);
1121 }
1122
1123 static void intel_crtc_enable_flip_done(struct intel_atomic_state *state,
1124                                         struct intel_crtc *crtc)
1125 {
1126         const struct intel_crtc_state *crtc_state =
1127                 intel_atomic_get_new_crtc_state(state, crtc);
1128         u8 update_planes = crtc_state->update_planes;
1129         const struct intel_plane_state *plane_state;
1130         struct intel_plane *plane;
1131         int i;
1132
1133         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1134                 if (plane->pipe == crtc->pipe &&
1135                     update_planes & BIT(plane->id))
1136                         plane->enable_flip_done(plane);
1137         }
1138 }
1139
1140 static void intel_crtc_disable_flip_done(struct intel_atomic_state *state,
1141                                          struct intel_crtc *crtc)
1142 {
1143         const struct intel_crtc_state *crtc_state =
1144                 intel_atomic_get_new_crtc_state(state, crtc);
1145         u8 update_planes = crtc_state->update_planes;
1146         const struct intel_plane_state *plane_state;
1147         struct intel_plane *plane;
1148         int i;
1149
1150         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
1151                 if (plane->pipe == crtc->pipe &&
1152                     update_planes & BIT(plane->id))
1153                         plane->disable_flip_done(plane);
1154         }
1155 }
1156
1157 static void intel_crtc_async_flip_disable_wa(struct intel_atomic_state *state,
1158                                              struct intel_crtc *crtc)
1159 {
1160         const struct intel_crtc_state *old_crtc_state =
1161                 intel_atomic_get_old_crtc_state(state, crtc);
1162         const struct intel_crtc_state *new_crtc_state =
1163                 intel_atomic_get_new_crtc_state(state, crtc);
1164         u8 disable_async_flip_planes = old_crtc_state->async_flip_planes &
1165                                        ~new_crtc_state->async_flip_planes;
1166         const struct intel_plane_state *old_plane_state;
1167         struct intel_plane *plane;
1168         bool need_vbl_wait = false;
1169         int i;
1170
1171         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
1172                 if (plane->need_async_flip_disable_wa &&
1173                     plane->pipe == crtc->pipe &&
1174                     disable_async_flip_planes & BIT(plane->id)) {
1175                         /*
1176                          * Apart from the async flip bit we want to
1177                          * preserve the old state for the plane.
1178                          */
1179                         plane->async_flip(plane, old_crtc_state,
1180                                           old_plane_state, false);
1181                         need_vbl_wait = true;
1182                 }
1183         }
1184
1185         if (need_vbl_wait)
1186                 intel_crtc_wait_for_next_vblank(crtc);
1187 }
1188
1189 static void intel_pre_plane_update(struct intel_atomic_state *state,
1190                                    struct intel_crtc *crtc)
1191 {
1192         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1193         const struct intel_crtc_state *old_crtc_state =
1194                 intel_atomic_get_old_crtc_state(state, crtc);
1195         const struct intel_crtc_state *new_crtc_state =
1196                 intel_atomic_get_new_crtc_state(state, crtc);
1197         enum pipe pipe = crtc->pipe;
1198
1199         intel_drrs_deactivate(old_crtc_state);
1200
1201         intel_psr_pre_plane_update(state, crtc);
1202
1203         if (hsw_ips_pre_update(state, crtc))
1204                 intel_crtc_wait_for_next_vblank(crtc);
1205
1206         if (intel_fbc_pre_update(state, crtc))
1207                 intel_crtc_wait_for_next_vblank(crtc);
1208
1209         if (!needs_async_flip_vtd_wa(old_crtc_state) &&
1210             needs_async_flip_vtd_wa(new_crtc_state))
1211                 intel_async_flip_vtd_wa(dev_priv, pipe, true);
1212
1213         /* Display WA 827 */
1214         if (!needs_nv12_wa(old_crtc_state) &&
1215             needs_nv12_wa(new_crtc_state))
1216                 skl_wa_827(dev_priv, pipe, true);
1217
1218         /* Wa_2006604312:icl,ehl */
1219         if (!needs_scalerclk_wa(old_crtc_state) &&
1220             needs_scalerclk_wa(new_crtc_state))
1221                 icl_wa_scalerclkgating(dev_priv, pipe, true);
1222
1223         /* Wa_1604331009:icl,jsl,ehl */
1224         if (!needs_cursorclk_wa(old_crtc_state) &&
1225             needs_cursorclk_wa(new_crtc_state))
1226                 icl_wa_cursorclkgating(dev_priv, pipe, true);
1227
1228         /*
1229          * Vblank time updates from the shadow to live plane control register
1230          * are blocked if the memory self-refresh mode is active at that
1231          * moment. So to make sure the plane gets truly disabled, disable
1232          * first the self-refresh mode. The self-refresh enable bit in turn
1233          * will be checked/applied by the HW only at the next frame start
1234          * event which is after the vblank start event, so we need to have a
1235          * wait-for-vblank between disabling the plane and the pipe.
1236          */
1237         if (HAS_GMCH(dev_priv) && old_crtc_state->hw.active &&
1238             new_crtc_state->disable_cxsr && intel_set_memory_cxsr(dev_priv, false))
1239                 intel_crtc_wait_for_next_vblank(crtc);
1240
1241         /*
1242          * IVB workaround: must disable low power watermarks for at least
1243          * one frame before enabling scaling.  LP watermarks can be re-enabled
1244          * when scaling is disabled.
1245          *
1246          * WaCxSRDisabledForSpriteScaling:ivb
1247          */
1248         if (old_crtc_state->hw.active &&
1249             new_crtc_state->disable_lp_wm && ilk_disable_lp_wm(dev_priv))
1250                 intel_crtc_wait_for_next_vblank(crtc);
1251
1252         /*
1253          * If we're doing a modeset we don't need to do any
1254          * pre-vblank watermark programming here.
1255          */
1256         if (!intel_crtc_needs_modeset(new_crtc_state)) {
1257                 /*
1258                  * For platforms that support atomic watermarks, program the
1259                  * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
1260                  * will be the intermediate values that are safe for both pre- and
1261                  * post- vblank; when vblank happens, the 'active' values will be set
1262                  * to the final 'target' values and we'll do this again to get the
1263                  * optimal watermarks.  For gen9+ platforms, the values we program here
1264                  * will be the final target values which will get automatically latched
1265                  * at vblank time; no further programming will be necessary.
1266                  *
1267                  * If a platform hasn't been transitioned to atomic watermarks yet,
1268                  * we'll continue to update watermarks the old way, if flags tell
1269                  * us to.
1270                  */
1271                 if (!intel_initial_watermarks(state, crtc))
1272                         if (new_crtc_state->update_wm_pre)
1273                                 intel_update_watermarks(dev_priv);
1274         }
1275
1276         /*
1277          * Gen2 reports pipe underruns whenever all planes are disabled.
1278          * So disable underrun reporting before all the planes get disabled.
1279          *
1280          * We do this after .initial_watermarks() so that we have a
1281          * chance of catching underruns with the intermediate watermarks
1282          * vs. the old plane configuration.
1283          */
1284         if (DISPLAY_VER(dev_priv) == 2 && planes_disabling(old_crtc_state, new_crtc_state))
1285                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
1286
1287         /*
1288          * WA for platforms where async address update enable bit
1289          * is double buffered and only latched at start of vblank.
1290          */
1291         if (old_crtc_state->async_flip_planes & ~new_crtc_state->async_flip_planes)
1292                 intel_crtc_async_flip_disable_wa(state, crtc);
1293 }
1294
1295 static void intel_crtc_disable_planes(struct intel_atomic_state *state,
1296                                       struct intel_crtc *crtc)
1297 {
1298         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1299         const struct intel_crtc_state *new_crtc_state =
1300                 intel_atomic_get_new_crtc_state(state, crtc);
1301         unsigned int update_mask = new_crtc_state->update_planes;
1302         const struct intel_plane_state *old_plane_state;
1303         struct intel_plane *plane;
1304         unsigned fb_bits = 0;
1305         int i;
1306
1307         intel_crtc_dpms_overlay_disable(crtc);
1308
1309         for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
1310                 if (crtc->pipe != plane->pipe ||
1311                     !(update_mask & BIT(plane->id)))
1312                         continue;
1313
1314                 intel_plane_disable_arm(plane, new_crtc_state);
1315
1316                 if (old_plane_state->uapi.visible)
1317                         fb_bits |= plane->frontbuffer_bit;
1318         }
1319
1320         intel_frontbuffer_flip(dev_priv, fb_bits);
1321 }
1322
1323 /*
1324  * intel_connector_primary_encoder - get the primary encoder for a connector
1325  * @connector: connector for which to return the encoder
1326  *
1327  * Returns the primary encoder for a connector. There is a 1:1 mapping from
1328  * all connectors to their encoder, except for DP-MST connectors which have
1329  * both a virtual and a primary encoder. These DP-MST primary encoders can be
1330  * pointed to by as many DP-MST connectors as there are pipes.
1331  */
1332 static struct intel_encoder *
1333 intel_connector_primary_encoder(struct intel_connector *connector)
1334 {
1335         struct intel_encoder *encoder;
1336
1337         if (connector->mst_port)
1338                 return &dp_to_dig_port(connector->mst_port)->base;
1339
1340         encoder = intel_attached_encoder(connector);
1341         drm_WARN_ON(connector->base.dev, !encoder);
1342
1343         return encoder;
1344 }
1345
1346 static void intel_encoders_update_prepare(struct intel_atomic_state *state)
1347 {
1348         struct drm_i915_private *i915 = to_i915(state->base.dev);
1349         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1350         struct intel_crtc *crtc;
1351         struct drm_connector_state *new_conn_state;
1352         struct drm_connector *connector;
1353         int i;
1354
1355         /*
1356          * Make sure the DPLL state is up-to-date for fastset TypeC ports after non-blocking commits.
1357          * TODO: Update the DPLL state for all cases in the encoder->update_prepare() hook.
1358          */
1359         if (i915->display.dpll.mgr) {
1360                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
1361                         if (intel_crtc_needs_modeset(new_crtc_state))
1362                                 continue;
1363
1364                         new_crtc_state->shared_dpll = old_crtc_state->shared_dpll;
1365                         new_crtc_state->dpll_hw_state = old_crtc_state->dpll_hw_state;
1366                 }
1367         }
1368
1369         if (!state->modeset)
1370                 return;
1371
1372         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
1373                                         i) {
1374                 struct intel_connector *intel_connector;
1375                 struct intel_encoder *encoder;
1376                 struct intel_crtc *crtc;
1377
1378                 if (!intel_connector_needs_modeset(state, connector))
1379                         continue;
1380
1381                 intel_connector = to_intel_connector(connector);
1382                 encoder = intel_connector_primary_encoder(intel_connector);
1383                 if (!encoder->update_prepare)
1384                         continue;
1385
1386                 crtc = new_conn_state->crtc ?
1387                         to_intel_crtc(new_conn_state->crtc) : NULL;
1388                 encoder->update_prepare(state, encoder, crtc);
1389         }
1390 }
1391
1392 static void intel_encoders_update_complete(struct intel_atomic_state *state)
1393 {
1394         struct drm_connector_state *new_conn_state;
1395         struct drm_connector *connector;
1396         int i;
1397
1398         if (!state->modeset)
1399                 return;
1400
1401         for_each_new_connector_in_state(&state->base, connector, new_conn_state,
1402                                         i) {
1403                 struct intel_connector *intel_connector;
1404                 struct intel_encoder *encoder;
1405                 struct intel_crtc *crtc;
1406
1407                 if (!intel_connector_needs_modeset(state, connector))
1408                         continue;
1409
1410                 intel_connector = to_intel_connector(connector);
1411                 encoder = intel_connector_primary_encoder(intel_connector);
1412                 if (!encoder->update_complete)
1413                         continue;
1414
1415                 crtc = new_conn_state->crtc ?
1416                         to_intel_crtc(new_conn_state->crtc) : NULL;
1417                 encoder->update_complete(state, encoder, crtc);
1418         }
1419 }
1420
1421 static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state,
1422                                           struct intel_crtc *crtc)
1423 {
1424         const struct intel_crtc_state *crtc_state =
1425                 intel_atomic_get_new_crtc_state(state, crtc);
1426         const struct drm_connector_state *conn_state;
1427         struct drm_connector *conn;
1428         int i;
1429
1430         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
1431                 struct intel_encoder *encoder =
1432                         to_intel_encoder(conn_state->best_encoder);
1433
1434                 if (conn_state->crtc != &crtc->base)
1435                         continue;
1436
1437                 if (encoder->pre_pll_enable)
1438                         encoder->pre_pll_enable(state, encoder,
1439                                                 crtc_state, conn_state);
1440         }
1441 }
1442
1443 static void intel_encoders_pre_enable(struct intel_atomic_state *state,
1444                                       struct intel_crtc *crtc)
1445 {
1446         const struct intel_crtc_state *crtc_state =
1447                 intel_atomic_get_new_crtc_state(state, crtc);
1448         const struct drm_connector_state *conn_state;
1449         struct drm_connector *conn;
1450         int i;
1451
1452         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
1453                 struct intel_encoder *encoder =
1454                         to_intel_encoder(conn_state->best_encoder);
1455
1456                 if (conn_state->crtc != &crtc->base)
1457                         continue;
1458
1459                 if (encoder->pre_enable)
1460                         encoder->pre_enable(state, encoder,
1461                                             crtc_state, conn_state);
1462         }
1463 }
1464
1465 static void intel_encoders_enable(struct intel_atomic_state *state,
1466                                   struct intel_crtc *crtc)
1467 {
1468         const struct intel_crtc_state *crtc_state =
1469                 intel_atomic_get_new_crtc_state(state, crtc);
1470         const struct drm_connector_state *conn_state;
1471         struct drm_connector *conn;
1472         int i;
1473
1474         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
1475                 struct intel_encoder *encoder =
1476                         to_intel_encoder(conn_state->best_encoder);
1477
1478                 if (conn_state->crtc != &crtc->base)
1479                         continue;
1480
1481                 if (encoder->enable)
1482                         encoder->enable(state, encoder,
1483                                         crtc_state, conn_state);
1484                 intel_opregion_notify_encoder(encoder, true);
1485         }
1486 }
1487
1488 static void intel_encoders_disable(struct intel_atomic_state *state,
1489                                    struct intel_crtc *crtc)
1490 {
1491         const struct intel_crtc_state *old_crtc_state =
1492                 intel_atomic_get_old_crtc_state(state, crtc);
1493         const struct drm_connector_state *old_conn_state;
1494         struct drm_connector *conn;
1495         int i;
1496
1497         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
1498                 struct intel_encoder *encoder =
1499                         to_intel_encoder(old_conn_state->best_encoder);
1500
1501                 if (old_conn_state->crtc != &crtc->base)
1502                         continue;
1503
1504                 intel_opregion_notify_encoder(encoder, false);
1505                 if (encoder->disable)
1506                         encoder->disable(state, encoder,
1507                                          old_crtc_state, old_conn_state);
1508         }
1509 }
1510
1511 static void intel_encoders_post_disable(struct intel_atomic_state *state,
1512                                         struct intel_crtc *crtc)
1513 {
1514         const struct intel_crtc_state *old_crtc_state =
1515                 intel_atomic_get_old_crtc_state(state, crtc);
1516         const struct drm_connector_state *old_conn_state;
1517         struct drm_connector *conn;
1518         int i;
1519
1520         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
1521                 struct intel_encoder *encoder =
1522                         to_intel_encoder(old_conn_state->best_encoder);
1523
1524                 if (old_conn_state->crtc != &crtc->base)
1525                         continue;
1526
1527                 if (encoder->post_disable)
1528                         encoder->post_disable(state, encoder,
1529                                               old_crtc_state, old_conn_state);
1530         }
1531 }
1532
1533 static void intel_encoders_post_pll_disable(struct intel_atomic_state *state,
1534                                             struct intel_crtc *crtc)
1535 {
1536         const struct intel_crtc_state *old_crtc_state =
1537                 intel_atomic_get_old_crtc_state(state, crtc);
1538         const struct drm_connector_state *old_conn_state;
1539         struct drm_connector *conn;
1540         int i;
1541
1542         for_each_old_connector_in_state(&state->base, conn, old_conn_state, i) {
1543                 struct intel_encoder *encoder =
1544                         to_intel_encoder(old_conn_state->best_encoder);
1545
1546                 if (old_conn_state->crtc != &crtc->base)
1547                         continue;
1548
1549                 if (encoder->post_pll_disable)
1550                         encoder->post_pll_disable(state, encoder,
1551                                                   old_crtc_state, old_conn_state);
1552         }
1553 }
1554
1555 static void intel_encoders_update_pipe(struct intel_atomic_state *state,
1556                                        struct intel_crtc *crtc)
1557 {
1558         const struct intel_crtc_state *crtc_state =
1559                 intel_atomic_get_new_crtc_state(state, crtc);
1560         const struct drm_connector_state *conn_state;
1561         struct drm_connector *conn;
1562         int i;
1563
1564         for_each_new_connector_in_state(&state->base, conn, conn_state, i) {
1565                 struct intel_encoder *encoder =
1566                         to_intel_encoder(conn_state->best_encoder);
1567
1568                 if (conn_state->crtc != &crtc->base)
1569                         continue;
1570
1571                 if (encoder->update_pipe)
1572                         encoder->update_pipe(state, encoder,
1573                                              crtc_state, conn_state);
1574         }
1575 }
1576
1577 static void intel_disable_primary_plane(const struct intel_crtc_state *crtc_state)
1578 {
1579         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1580         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
1581
1582         plane->disable_arm(plane, crtc_state);
1583 }
1584
1585 static void ilk_configure_cpu_transcoder(const struct intel_crtc_state *crtc_state)
1586 {
1587         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1588         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
1589
1590         if (crtc_state->has_pch_encoder) {
1591                 intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder,
1592                                                &crtc_state->fdi_m_n);
1593         } else if (intel_crtc_has_dp_encoder(crtc_state)) {
1594                 intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder,
1595                                                &crtc_state->dp_m_n);
1596                 intel_cpu_transcoder_set_m2_n2(crtc, cpu_transcoder,
1597                                                &crtc_state->dp_m2_n2);
1598         }
1599
1600         intel_set_transcoder_timings(crtc_state);
1601
1602         ilk_set_pipeconf(crtc_state);
1603 }
1604
1605 static void ilk_crtc_enable(struct intel_atomic_state *state,
1606                             struct intel_crtc *crtc)
1607 {
1608         const struct intel_crtc_state *new_crtc_state =
1609                 intel_atomic_get_new_crtc_state(state, crtc);
1610         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1611         enum pipe pipe = crtc->pipe;
1612
1613         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
1614                 return;
1615
1616         /*
1617          * Sometimes spurious CPU pipe underruns happen during FDI
1618          * training, at least with VGA+HDMI cloning. Suppress them.
1619          *
1620          * On ILK we get an occasional spurious CPU pipe underruns
1621          * between eDP port A enable and vdd enable. Also PCH port
1622          * enable seems to result in the occasional CPU pipe underrun.
1623          *
1624          * Spurious PCH underruns also occur during PCH enabling.
1625          */
1626         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
1627         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
1628
1629         ilk_configure_cpu_transcoder(new_crtc_state);
1630
1631         intel_set_pipe_src_size(new_crtc_state);
1632
1633         crtc->active = true;
1634
1635         intel_encoders_pre_enable(state, crtc);
1636
1637         if (new_crtc_state->has_pch_encoder) {
1638                 ilk_pch_pre_enable(state, crtc);
1639         } else {
1640                 assert_fdi_tx_disabled(dev_priv, pipe);
1641                 assert_fdi_rx_disabled(dev_priv, pipe);
1642         }
1643
1644         ilk_pfit_enable(new_crtc_state);
1645
1646         /*
1647          * On ILK+ LUT must be loaded before the pipe is running but with
1648          * clocks enabled
1649          */
1650         intel_color_load_luts(new_crtc_state);
1651         intel_color_commit_noarm(new_crtc_state);
1652         intel_color_commit_arm(new_crtc_state);
1653         /* update DSPCNTR to configure gamma for pipe bottom color */
1654         intel_disable_primary_plane(new_crtc_state);
1655
1656         intel_initial_watermarks(state, crtc);
1657         intel_enable_transcoder(new_crtc_state);
1658
1659         if (new_crtc_state->has_pch_encoder)
1660                 ilk_pch_enable(state, crtc);
1661
1662         intel_crtc_vblank_on(new_crtc_state);
1663
1664         intel_encoders_enable(state, crtc);
1665
1666         if (HAS_PCH_CPT(dev_priv))
1667                 intel_wait_for_pipe_scanline_moving(crtc);
1668
1669         /*
1670          * Must wait for vblank to avoid spurious PCH FIFO underruns.
1671          * And a second vblank wait is needed at least on ILK with
1672          * some interlaced HDMI modes. Let's do the double wait always
1673          * in case there are more corner cases we don't know about.
1674          */
1675         if (new_crtc_state->has_pch_encoder) {
1676                 intel_crtc_wait_for_next_vblank(crtc);
1677                 intel_crtc_wait_for_next_vblank(crtc);
1678         }
1679         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
1680         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
1681 }
1682
1683 static void glk_pipe_scaler_clock_gating_wa(struct drm_i915_private *dev_priv,
1684                                             enum pipe pipe, bool apply)
1685 {
1686         u32 val = intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe));
1687         u32 mask = DPF_GATING_DIS | DPF_RAM_GATING_DIS | DPFR_GATING_DIS;
1688
1689         if (apply)
1690                 val |= mask;
1691         else
1692                 val &= ~mask;
1693
1694         intel_de_write(dev_priv, CLKGATE_DIS_PSL(pipe), val);
1695 }
1696
1697 static void hsw_set_linetime_wm(const struct intel_crtc_state *crtc_state)
1698 {
1699         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1700         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1701
1702         intel_de_write(dev_priv, WM_LINETIME(crtc->pipe),
1703                        HSW_LINETIME(crtc_state->linetime) |
1704                        HSW_IPS_LINETIME(crtc_state->ips_linetime));
1705 }
1706
1707 static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
1708 {
1709         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1710         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1711         enum transcoder transcoder = crtc_state->cpu_transcoder;
1712         i915_reg_t reg = DISPLAY_VER(dev_priv) >= 14 ? MTL_CHICKEN_TRANS(transcoder) :
1713                          CHICKEN_TRANS(transcoder);
1714
1715         intel_de_rmw(dev_priv, reg,
1716                      HSW_FRAME_START_DELAY_MASK,
1717                      HSW_FRAME_START_DELAY(crtc_state->framestart_delay - 1));
1718 }
1719
1720 static void icl_ddi_bigjoiner_pre_enable(struct intel_atomic_state *state,
1721                                          const struct intel_crtc_state *crtc_state)
1722 {
1723         struct intel_crtc *master_crtc = intel_master_crtc(crtc_state);
1724
1725         /*
1726          * Enable sequence steps 1-7 on bigjoiner master
1727          */
1728         if (intel_crtc_is_bigjoiner_slave(crtc_state))
1729                 intel_encoders_pre_pll_enable(state, master_crtc);
1730
1731         if (crtc_state->shared_dpll)
1732                 intel_enable_shared_dpll(crtc_state);
1733
1734         if (intel_crtc_is_bigjoiner_slave(crtc_state))
1735                 intel_encoders_pre_enable(state, master_crtc);
1736 }
1737
1738 static void hsw_configure_cpu_transcoder(const struct intel_crtc_state *crtc_state)
1739 {
1740         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1741         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1742         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
1743
1744         if (crtc_state->has_pch_encoder) {
1745                 intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder,
1746                                                &crtc_state->fdi_m_n);
1747         } else if (intel_crtc_has_dp_encoder(crtc_state)) {
1748                 intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder,
1749                                                &crtc_state->dp_m_n);
1750                 intel_cpu_transcoder_set_m2_n2(crtc, cpu_transcoder,
1751                                                &crtc_state->dp_m2_n2);
1752         }
1753
1754         intel_set_transcoder_timings(crtc_state);
1755
1756         if (cpu_transcoder != TRANSCODER_EDP)
1757                 intel_de_write(dev_priv, TRANS_MULT(cpu_transcoder),
1758                                crtc_state->pixel_multiplier - 1);
1759
1760         hsw_set_frame_start_delay(crtc_state);
1761
1762         hsw_set_transconf(crtc_state);
1763 }
1764
1765 static void hsw_crtc_enable(struct intel_atomic_state *state,
1766                             struct intel_crtc *crtc)
1767 {
1768         const struct intel_crtc_state *new_crtc_state =
1769                 intel_atomic_get_new_crtc_state(state, crtc);
1770         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1771         enum pipe pipe = crtc->pipe, hsw_workaround_pipe;
1772         enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder;
1773         bool psl_clkgate_wa;
1774
1775         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
1776                 return;
1777
1778         intel_dmc_enable_pipe(dev_priv, crtc->pipe);
1779
1780         if (!new_crtc_state->bigjoiner_pipes) {
1781                 intel_encoders_pre_pll_enable(state, crtc);
1782
1783                 if (new_crtc_state->shared_dpll)
1784                         intel_enable_shared_dpll(new_crtc_state);
1785
1786                 intel_encoders_pre_enable(state, crtc);
1787         } else {
1788                 icl_ddi_bigjoiner_pre_enable(state, new_crtc_state);
1789         }
1790
1791         intel_dsc_enable(new_crtc_state);
1792
1793         if (DISPLAY_VER(dev_priv) >= 13)
1794                 intel_uncompressed_joiner_enable(new_crtc_state);
1795
1796         intel_set_pipe_src_size(new_crtc_state);
1797         if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
1798                 bdw_set_pipe_misc(new_crtc_state);
1799
1800         if (!intel_crtc_is_bigjoiner_slave(new_crtc_state) &&
1801             !transcoder_is_dsi(cpu_transcoder))
1802                 hsw_configure_cpu_transcoder(new_crtc_state);
1803
1804         crtc->active = true;
1805
1806         /* Display WA #1180: WaDisableScalarClockGating: glk */
1807         psl_clkgate_wa = DISPLAY_VER(dev_priv) == 10 &&
1808                 new_crtc_state->pch_pfit.enabled;
1809         if (psl_clkgate_wa)
1810                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, true);
1811
1812         if (DISPLAY_VER(dev_priv) >= 9)
1813                 skl_pfit_enable(new_crtc_state);
1814         else
1815                 ilk_pfit_enable(new_crtc_state);
1816
1817         /*
1818          * On ILK+ LUT must be loaded before the pipe is running but with
1819          * clocks enabled
1820          */
1821         intel_color_load_luts(new_crtc_state);
1822         intel_color_commit_noarm(new_crtc_state);
1823         intel_color_commit_arm(new_crtc_state);
1824         /* update DSPCNTR to configure gamma/csc for pipe bottom color */
1825         if (DISPLAY_VER(dev_priv) < 9)
1826                 intel_disable_primary_plane(new_crtc_state);
1827
1828         hsw_set_linetime_wm(new_crtc_state);
1829
1830         if (DISPLAY_VER(dev_priv) >= 11)
1831                 icl_set_pipe_chicken(new_crtc_state);
1832
1833         intel_initial_watermarks(state, crtc);
1834
1835         if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
1836                 intel_crtc_vblank_on(new_crtc_state);
1837
1838         intel_encoders_enable(state, crtc);
1839
1840         if (psl_clkgate_wa) {
1841                 intel_crtc_wait_for_next_vblank(crtc);
1842                 glk_pipe_scaler_clock_gating_wa(dev_priv, pipe, false);
1843         }
1844
1845         /* If we change the relative order between pipe/planes enabling, we need
1846          * to change the workaround. */
1847         hsw_workaround_pipe = new_crtc_state->hsw_workaround_pipe;
1848         if (IS_HASWELL(dev_priv) && hsw_workaround_pipe != INVALID_PIPE) {
1849                 struct intel_crtc *wa_crtc;
1850
1851                 wa_crtc = intel_crtc_for_pipe(dev_priv, hsw_workaround_pipe);
1852
1853                 intel_crtc_wait_for_next_vblank(wa_crtc);
1854                 intel_crtc_wait_for_next_vblank(wa_crtc);
1855         }
1856 }
1857
1858 void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state)
1859 {
1860         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
1861         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1862         enum pipe pipe = crtc->pipe;
1863
1864         /* To avoid upsetting the power well on haswell only disable the pfit if
1865          * it's in use. The hw state code will make sure we get this right. */
1866         if (!old_crtc_state->pch_pfit.enabled)
1867                 return;
1868
1869         intel_de_write_fw(dev_priv, PF_CTL(pipe), 0);
1870         intel_de_write_fw(dev_priv, PF_WIN_POS(pipe), 0);
1871         intel_de_write_fw(dev_priv, PF_WIN_SZ(pipe), 0);
1872 }
1873
1874 static void ilk_crtc_disable(struct intel_atomic_state *state,
1875                              struct intel_crtc *crtc)
1876 {
1877         const struct intel_crtc_state *old_crtc_state =
1878                 intel_atomic_get_old_crtc_state(state, crtc);
1879         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1880         enum pipe pipe = crtc->pipe;
1881
1882         /*
1883          * Sometimes spurious CPU pipe underruns happen when the
1884          * pipe is already disabled, but FDI RX/TX is still enabled.
1885          * Happens at least with VGA+HDMI cloning. Suppress them.
1886          */
1887         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
1888         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
1889
1890         intel_encoders_disable(state, crtc);
1891
1892         intel_crtc_vblank_off(old_crtc_state);
1893
1894         intel_disable_transcoder(old_crtc_state);
1895
1896         ilk_pfit_disable(old_crtc_state);
1897
1898         if (old_crtc_state->has_pch_encoder)
1899                 ilk_pch_disable(state, crtc);
1900
1901         intel_encoders_post_disable(state, crtc);
1902
1903         if (old_crtc_state->has_pch_encoder)
1904                 ilk_pch_post_disable(state, crtc);
1905
1906         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
1907         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
1908 }
1909
1910 static void hsw_crtc_disable(struct intel_atomic_state *state,
1911                              struct intel_crtc *crtc)
1912 {
1913         const struct intel_crtc_state *old_crtc_state =
1914                 intel_atomic_get_old_crtc_state(state, crtc);
1915         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1916
1917         /*
1918          * FIXME collapse everything to one hook.
1919          * Need care with mst->ddi interactions.
1920          */
1921         if (!intel_crtc_is_bigjoiner_slave(old_crtc_state)) {
1922                 intel_encoders_disable(state, crtc);
1923                 intel_encoders_post_disable(state, crtc);
1924         }
1925
1926         intel_dmc_disable_pipe(i915, crtc->pipe);
1927 }
1928
1929 static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state)
1930 {
1931         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1932         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1933
1934         if (!crtc_state->gmch_pfit.control)
1935                 return;
1936
1937         /*
1938          * The panel fitter should only be adjusted whilst the pipe is disabled,
1939          * according to register description and PRM.
1940          */
1941         drm_WARN_ON(&dev_priv->drm,
1942                     intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE);
1943         assert_transcoder_disabled(dev_priv, crtc_state->cpu_transcoder);
1944
1945         intel_de_write(dev_priv, PFIT_PGM_RATIOS,
1946                        crtc_state->gmch_pfit.pgm_ratios);
1947         intel_de_write(dev_priv, PFIT_CONTROL, crtc_state->gmch_pfit.control);
1948
1949         /* Border color in case we don't scale up to the full screen. Black by
1950          * default, change to something else for debugging. */
1951         intel_de_write(dev_priv, BCLRPAT(crtc->pipe), 0);
1952 }
1953
1954 bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
1955 {
1956         if (phy == PHY_NONE)
1957                 return false;
1958         else if (IS_ALDERLAKE_S(dev_priv))
1959                 return phy <= PHY_E;
1960         else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv))
1961                 return phy <= PHY_D;
1962         else if (IS_JSL_EHL(dev_priv))
1963                 return phy <= PHY_C;
1964         else if (IS_ALDERLAKE_P(dev_priv) || IS_DISPLAY_VER(dev_priv, 11, 12))
1965                 return phy <= PHY_B;
1966         else
1967                 /*
1968                  * DG2 outputs labelled as "combo PHY" in the bspec use
1969                  * SNPS PHYs with completely different programming,
1970                  * hence we always return false here.
1971                  */
1972                 return false;
1973 }
1974
1975 bool intel_phy_is_tc(struct drm_i915_private *dev_priv, enum phy phy)
1976 {
1977         if (IS_DG2(dev_priv))
1978                 /* DG2's "TC1" output uses a SNPS PHY */
1979                 return false;
1980         else if (IS_ALDERLAKE_P(dev_priv))
1981                 return phy >= PHY_F && phy <= PHY_I;
1982         else if (IS_TIGERLAKE(dev_priv))
1983                 return phy >= PHY_D && phy <= PHY_I;
1984         else if (IS_ICELAKE(dev_priv))
1985                 return phy >= PHY_C && phy <= PHY_F;
1986         else
1987                 return false;
1988 }
1989
1990 bool intel_phy_is_snps(struct drm_i915_private *dev_priv, enum phy phy)
1991 {
1992         if (phy == PHY_NONE)
1993                 return false;
1994         else if (IS_DG2(dev_priv))
1995                 /*
1996                  * All four "combo" ports and the TC1 port (PHY E) use
1997                  * Synopsis PHYs.
1998                  */
1999                 return phy <= PHY_E;
2000
2001         return false;
2002 }
2003
2004 enum phy intel_port_to_phy(struct drm_i915_private *i915, enum port port)
2005 {
2006         if (DISPLAY_VER(i915) >= 13 && port >= PORT_D_XELPD)
2007                 return PHY_D + port - PORT_D_XELPD;
2008         else if (DISPLAY_VER(i915) >= 13 && port >= PORT_TC1)
2009                 return PHY_F + port - PORT_TC1;
2010         else if (IS_ALDERLAKE_S(i915) && port >= PORT_TC1)
2011                 return PHY_B + port - PORT_TC1;
2012         else if ((IS_DG1(i915) || IS_ROCKETLAKE(i915)) && port >= PORT_TC1)
2013                 return PHY_C + port - PORT_TC1;
2014         else if (IS_JSL_EHL(i915) && port == PORT_D)
2015                 return PHY_A;
2016
2017         return PHY_A + port - PORT_A;
2018 }
2019
2020 enum tc_port intel_port_to_tc(struct drm_i915_private *dev_priv, enum port port)
2021 {
2022         if (!intel_phy_is_tc(dev_priv, intel_port_to_phy(dev_priv, port)))
2023                 return TC_PORT_NONE;
2024
2025         if (DISPLAY_VER(dev_priv) >= 12)
2026                 return TC_PORT_1 + port - PORT_TC1;
2027         else
2028                 return TC_PORT_1 + port - PORT_C;
2029 }
2030
2031 enum intel_display_power_domain
2032 intel_aux_power_domain(struct intel_digital_port *dig_port)
2033 {
2034         struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
2035
2036         if (intel_tc_port_in_tbt_alt_mode(dig_port))
2037                 return intel_display_power_tbt_aux_domain(i915, dig_port->aux_ch);
2038
2039         return intel_display_power_legacy_aux_domain(i915, dig_port->aux_ch);
2040 }
2041
2042 static void get_crtc_power_domains(struct intel_crtc_state *crtc_state,
2043                                    struct intel_power_domain_mask *mask)
2044 {
2045         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2046         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2047         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2048         struct drm_encoder *encoder;
2049         enum pipe pipe = crtc->pipe;
2050
2051         bitmap_zero(mask->bits, POWER_DOMAIN_NUM);
2052
2053         if (!crtc_state->hw.active)
2054                 return;
2055
2056         set_bit(POWER_DOMAIN_PIPE(pipe), mask->bits);
2057         set_bit(POWER_DOMAIN_TRANSCODER(cpu_transcoder), mask->bits);
2058         if (crtc_state->pch_pfit.enabled ||
2059             crtc_state->pch_pfit.force_thru)
2060                 set_bit(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe), mask->bits);
2061
2062         drm_for_each_encoder_mask(encoder, &dev_priv->drm,
2063                                   crtc_state->uapi.encoder_mask) {
2064                 struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
2065
2066                 set_bit(intel_encoder->power_domain, mask->bits);
2067         }
2068
2069         if (HAS_DDI(dev_priv) && crtc_state->has_audio)
2070                 set_bit(POWER_DOMAIN_AUDIO_MMIO, mask->bits);
2071
2072         if (crtc_state->shared_dpll)
2073                 set_bit(POWER_DOMAIN_DISPLAY_CORE, mask->bits);
2074
2075         if (crtc_state->dsc.compression_enable)
2076                 set_bit(intel_dsc_power_domain(crtc, cpu_transcoder), mask->bits);
2077 }
2078
2079 void intel_modeset_get_crtc_power_domains(struct intel_crtc_state *crtc_state,
2080                                           struct intel_power_domain_mask *old_domains)
2081 {
2082         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2083         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2084         enum intel_display_power_domain domain;
2085         struct intel_power_domain_mask domains, new_domains;
2086
2087         get_crtc_power_domains(crtc_state, &domains);
2088
2089         bitmap_andnot(new_domains.bits,
2090                       domains.bits,
2091                       crtc->enabled_power_domains.mask.bits,
2092                       POWER_DOMAIN_NUM);
2093         bitmap_andnot(old_domains->bits,
2094                       crtc->enabled_power_domains.mask.bits,
2095                       domains.bits,
2096                       POWER_DOMAIN_NUM);
2097
2098         for_each_power_domain(domain, &new_domains)
2099                 intel_display_power_get_in_set(dev_priv,
2100                                                &crtc->enabled_power_domains,
2101                                                domain);
2102 }
2103
2104 void intel_modeset_put_crtc_power_domains(struct intel_crtc *crtc,
2105                                           struct intel_power_domain_mask *domains)
2106 {
2107         intel_display_power_put_mask_in_set(to_i915(crtc->base.dev),
2108                                             &crtc->enabled_power_domains,
2109                                             domains);
2110 }
2111
2112 static void i9xx_configure_cpu_transcoder(const struct intel_crtc_state *crtc_state)
2113 {
2114         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2115         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2116
2117         if (intel_crtc_has_dp_encoder(crtc_state)) {
2118                 intel_cpu_transcoder_set_m1_n1(crtc, cpu_transcoder,
2119                                                &crtc_state->dp_m_n);
2120                 intel_cpu_transcoder_set_m2_n2(crtc, cpu_transcoder,
2121                                                &crtc_state->dp_m2_n2);
2122         }
2123
2124         intel_set_transcoder_timings(crtc_state);
2125
2126         i9xx_set_pipeconf(crtc_state);
2127 }
2128
2129 static void valleyview_crtc_enable(struct intel_atomic_state *state,
2130                                    struct intel_crtc *crtc)
2131 {
2132         const struct intel_crtc_state *new_crtc_state =
2133                 intel_atomic_get_new_crtc_state(state, crtc);
2134         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2135         enum pipe pipe = crtc->pipe;
2136
2137         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
2138                 return;
2139
2140         i9xx_configure_cpu_transcoder(new_crtc_state);
2141
2142         intel_set_pipe_src_size(new_crtc_state);
2143
2144         intel_de_write(dev_priv, VLV_PIPE_MSA_MISC(pipe), 0);
2145
2146         if (IS_CHERRYVIEW(dev_priv) && pipe == PIPE_B) {
2147                 intel_de_write(dev_priv, CHV_BLEND(pipe), CHV_BLEND_LEGACY);
2148                 intel_de_write(dev_priv, CHV_CANVAS(pipe), 0);
2149         }
2150
2151         crtc->active = true;
2152
2153         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
2154
2155         intel_encoders_pre_pll_enable(state, crtc);
2156
2157         if (IS_CHERRYVIEW(dev_priv))
2158                 chv_enable_pll(new_crtc_state);
2159         else
2160                 vlv_enable_pll(new_crtc_state);
2161
2162         intel_encoders_pre_enable(state, crtc);
2163
2164         i9xx_pfit_enable(new_crtc_state);
2165
2166         intel_color_load_luts(new_crtc_state);
2167         intel_color_commit_noarm(new_crtc_state);
2168         intel_color_commit_arm(new_crtc_state);
2169         /* update DSPCNTR to configure gamma for pipe bottom color */
2170         intel_disable_primary_plane(new_crtc_state);
2171
2172         intel_initial_watermarks(state, crtc);
2173         intel_enable_transcoder(new_crtc_state);
2174
2175         intel_crtc_vblank_on(new_crtc_state);
2176
2177         intel_encoders_enable(state, crtc);
2178 }
2179
2180 static void i9xx_crtc_enable(struct intel_atomic_state *state,
2181                              struct intel_crtc *crtc)
2182 {
2183         const struct intel_crtc_state *new_crtc_state =
2184                 intel_atomic_get_new_crtc_state(state, crtc);
2185         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2186         enum pipe pipe = crtc->pipe;
2187
2188         if (drm_WARN_ON(&dev_priv->drm, crtc->active))
2189                 return;
2190
2191         i9xx_configure_cpu_transcoder(new_crtc_state);
2192
2193         intel_set_pipe_src_size(new_crtc_state);
2194
2195         crtc->active = true;
2196
2197         if (DISPLAY_VER(dev_priv) != 2)
2198                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
2199
2200         intel_encoders_pre_enable(state, crtc);
2201
2202         i9xx_enable_pll(new_crtc_state);
2203
2204         i9xx_pfit_enable(new_crtc_state);
2205
2206         intel_color_load_luts(new_crtc_state);
2207         intel_color_commit_noarm(new_crtc_state);
2208         intel_color_commit_arm(new_crtc_state);
2209         /* update DSPCNTR to configure gamma for pipe bottom color */
2210         intel_disable_primary_plane(new_crtc_state);
2211
2212         if (!intel_initial_watermarks(state, crtc))
2213                 intel_update_watermarks(dev_priv);
2214         intel_enable_transcoder(new_crtc_state);
2215
2216         intel_crtc_vblank_on(new_crtc_state);
2217
2218         intel_encoders_enable(state, crtc);
2219
2220         /* prevents spurious underruns */
2221         if (DISPLAY_VER(dev_priv) == 2)
2222                 intel_crtc_wait_for_next_vblank(crtc);
2223 }
2224
2225 static void i9xx_pfit_disable(const struct intel_crtc_state *old_crtc_state)
2226 {
2227         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
2228         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2229
2230         if (!old_crtc_state->gmch_pfit.control)
2231                 return;
2232
2233         assert_transcoder_disabled(dev_priv, old_crtc_state->cpu_transcoder);
2234
2235         drm_dbg_kms(&dev_priv->drm, "disabling pfit, current: 0x%08x\n",
2236                     intel_de_read(dev_priv, PFIT_CONTROL));
2237         intel_de_write(dev_priv, PFIT_CONTROL, 0);
2238 }
2239
2240 static void i9xx_crtc_disable(struct intel_atomic_state *state,
2241                               struct intel_crtc *crtc)
2242 {
2243         struct intel_crtc_state *old_crtc_state =
2244                 intel_atomic_get_old_crtc_state(state, crtc);
2245         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2246         enum pipe pipe = crtc->pipe;
2247
2248         /*
2249          * On gen2 planes are double buffered but the pipe isn't, so we must
2250          * wait for planes to fully turn off before disabling the pipe.
2251          */
2252         if (DISPLAY_VER(dev_priv) == 2)
2253                 intel_crtc_wait_for_next_vblank(crtc);
2254
2255         intel_encoders_disable(state, crtc);
2256
2257         intel_crtc_vblank_off(old_crtc_state);
2258
2259         intel_disable_transcoder(old_crtc_state);
2260
2261         i9xx_pfit_disable(old_crtc_state);
2262
2263         intel_encoders_post_disable(state, crtc);
2264
2265         if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DSI)) {
2266                 if (IS_CHERRYVIEW(dev_priv))
2267                         chv_disable_pll(dev_priv, pipe);
2268                 else if (IS_VALLEYVIEW(dev_priv))
2269                         vlv_disable_pll(dev_priv, pipe);
2270                 else
2271                         i9xx_disable_pll(old_crtc_state);
2272         }
2273
2274         intel_encoders_post_pll_disable(state, crtc);
2275
2276         if (DISPLAY_VER(dev_priv) != 2)
2277                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
2278
2279         if (!dev_priv->display.funcs.wm->initial_watermarks)
2280                 intel_update_watermarks(dev_priv);
2281
2282         /* clock the pipe down to 640x480@60 to potentially save power */
2283         if (IS_I830(dev_priv))
2284                 i830_enable_pipe(dev_priv, pipe);
2285 }
2286
2287
2288 /*
2289  * turn all crtc's off, but do not adjust state
2290  * This has to be paired with a call to intel_modeset_setup_hw_state.
2291  */
2292 int intel_display_suspend(struct drm_device *dev)
2293 {
2294         struct drm_i915_private *dev_priv = to_i915(dev);
2295         struct drm_atomic_state *state;
2296         int ret;
2297
2298         if (!HAS_DISPLAY(dev_priv))
2299                 return 0;
2300
2301         state = drm_atomic_helper_suspend(dev);
2302         ret = PTR_ERR_OR_ZERO(state);
2303         if (ret)
2304                 drm_err(&dev_priv->drm, "Suspending crtc's failed with %i\n",
2305                         ret);
2306         else
2307                 dev_priv->display.restore.modeset_state = state;
2308         return ret;
2309 }
2310
2311 void intel_encoder_destroy(struct drm_encoder *encoder)
2312 {
2313         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
2314
2315         drm_encoder_cleanup(encoder);
2316         kfree(intel_encoder);
2317 }
2318
2319 static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc)
2320 {
2321         const struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2322
2323         /* GDG double wide on either pipe, otherwise pipe A only */
2324         return DISPLAY_VER(dev_priv) < 4 &&
2325                 (crtc->pipe == PIPE_A || IS_I915G(dev_priv));
2326 }
2327
2328 static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state)
2329 {
2330         u32 pixel_rate = crtc_state->hw.pipe_mode.crtc_clock;
2331         struct drm_rect src;
2332
2333         /*
2334          * We only use IF-ID interlacing. If we ever use
2335          * PF-ID we'll need to adjust the pixel_rate here.
2336          */
2337
2338         if (!crtc_state->pch_pfit.enabled)
2339                 return pixel_rate;
2340
2341         drm_rect_init(&src, 0, 0,
2342                       drm_rect_width(&crtc_state->pipe_src) << 16,
2343                       drm_rect_height(&crtc_state->pipe_src) << 16);
2344
2345         return intel_adjusted_rate(&src, &crtc_state->pch_pfit.dst,
2346                                    pixel_rate);
2347 }
2348
2349 static void intel_mode_from_crtc_timings(struct drm_display_mode *mode,
2350                                          const struct drm_display_mode *timings)
2351 {
2352         mode->hdisplay = timings->crtc_hdisplay;
2353         mode->htotal = timings->crtc_htotal;
2354         mode->hsync_start = timings->crtc_hsync_start;
2355         mode->hsync_end = timings->crtc_hsync_end;
2356
2357         mode->vdisplay = timings->crtc_vdisplay;
2358         mode->vtotal = timings->crtc_vtotal;
2359         mode->vsync_start = timings->crtc_vsync_start;
2360         mode->vsync_end = timings->crtc_vsync_end;
2361
2362         mode->flags = timings->flags;
2363         mode->type = DRM_MODE_TYPE_DRIVER;
2364
2365         mode->clock = timings->crtc_clock;
2366
2367         drm_mode_set_name(mode);
2368 }
2369
2370 static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state)
2371 {
2372         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2373
2374         if (HAS_GMCH(dev_priv))
2375                 /* FIXME calculate proper pipe pixel rate for GMCH pfit */
2376                 crtc_state->pixel_rate =
2377                         crtc_state->hw.pipe_mode.crtc_clock;
2378         else
2379                 crtc_state->pixel_rate =
2380                         ilk_pipe_pixel_rate(crtc_state);
2381 }
2382
2383 static void intel_bigjoiner_adjust_timings(const struct intel_crtc_state *crtc_state,
2384                                            struct drm_display_mode *mode)
2385 {
2386         int num_pipes = intel_bigjoiner_num_pipes(crtc_state);
2387
2388         if (num_pipes < 2)
2389                 return;
2390
2391         mode->crtc_clock /= num_pipes;
2392         mode->crtc_hdisplay /= num_pipes;
2393         mode->crtc_hblank_start /= num_pipes;
2394         mode->crtc_hblank_end /= num_pipes;
2395         mode->crtc_hsync_start /= num_pipes;
2396         mode->crtc_hsync_end /= num_pipes;
2397         mode->crtc_htotal /= num_pipes;
2398 }
2399
2400 static void intel_splitter_adjust_timings(const struct intel_crtc_state *crtc_state,
2401                                           struct drm_display_mode *mode)
2402 {
2403         int overlap = crtc_state->splitter.pixel_overlap;
2404         int n = crtc_state->splitter.link_count;
2405
2406         if (!crtc_state->splitter.enable)
2407                 return;
2408
2409         /*
2410          * eDP MSO uses segment timings from EDID for transcoder
2411          * timings, but full mode for everything else.
2412          *
2413          * h_full = (h_segment - pixel_overlap) * link_count
2414          */
2415         mode->crtc_hdisplay = (mode->crtc_hdisplay - overlap) * n;
2416         mode->crtc_hblank_start = (mode->crtc_hblank_start - overlap) * n;
2417         mode->crtc_hblank_end = (mode->crtc_hblank_end - overlap) * n;
2418         mode->crtc_hsync_start = (mode->crtc_hsync_start - overlap) * n;
2419         mode->crtc_hsync_end = (mode->crtc_hsync_end - overlap) * n;
2420         mode->crtc_htotal = (mode->crtc_htotal - overlap) * n;
2421         mode->crtc_clock *= n;
2422 }
2423
2424 static void intel_crtc_readout_derived_state(struct intel_crtc_state *crtc_state)
2425 {
2426         struct drm_display_mode *mode = &crtc_state->hw.mode;
2427         struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
2428         struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
2429
2430         /*
2431          * Start with the adjusted_mode crtc timings, which
2432          * have been filled with the transcoder timings.
2433          */
2434         drm_mode_copy(pipe_mode, adjusted_mode);
2435
2436         /* Expand MSO per-segment transcoder timings to full */
2437         intel_splitter_adjust_timings(crtc_state, pipe_mode);
2438
2439         /*
2440          * We want the full numbers in adjusted_mode normal timings,
2441          * adjusted_mode crtc timings are left with the raw transcoder
2442          * timings.
2443          */
2444         intel_mode_from_crtc_timings(adjusted_mode, pipe_mode);
2445
2446         /* Populate the "user" mode with full numbers */
2447         drm_mode_copy(mode, pipe_mode);
2448         intel_mode_from_crtc_timings(mode, mode);
2449         mode->hdisplay = drm_rect_width(&crtc_state->pipe_src) *
2450                 (intel_bigjoiner_num_pipes(crtc_state) ?: 1);
2451         mode->vdisplay = drm_rect_height(&crtc_state->pipe_src);
2452
2453         /* Derive per-pipe timings in case bigjoiner is used */
2454         intel_bigjoiner_adjust_timings(crtc_state, pipe_mode);
2455         intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
2456
2457         intel_crtc_compute_pixel_rate(crtc_state);
2458 }
2459
2460 void intel_encoder_get_config(struct intel_encoder *encoder,
2461                               struct intel_crtc_state *crtc_state)
2462 {
2463         encoder->get_config(encoder, crtc_state);
2464
2465         intel_crtc_readout_derived_state(crtc_state);
2466 }
2467
2468 static void intel_bigjoiner_compute_pipe_src(struct intel_crtc_state *crtc_state)
2469 {
2470         int num_pipes = intel_bigjoiner_num_pipes(crtc_state);
2471         int width, height;
2472
2473         if (num_pipes < 2)
2474                 return;
2475
2476         width = drm_rect_width(&crtc_state->pipe_src);
2477         height = drm_rect_height(&crtc_state->pipe_src);
2478
2479         drm_rect_init(&crtc_state->pipe_src, 0, 0,
2480                       width / num_pipes, height);
2481 }
2482
2483 static int intel_crtc_compute_pipe_src(struct intel_crtc_state *crtc_state)
2484 {
2485         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2486         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2487
2488         intel_bigjoiner_compute_pipe_src(crtc_state);
2489
2490         /*
2491          * Pipe horizontal size must be even in:
2492          * - DVO ganged mode
2493          * - LVDS dual channel mode
2494          * - Double wide pipe
2495          */
2496         if (drm_rect_width(&crtc_state->pipe_src) & 1) {
2497                 if (crtc_state->double_wide) {
2498                         drm_dbg_kms(&i915->drm,
2499                                     "[CRTC:%d:%s] Odd pipe source width not supported with double wide pipe\n",
2500                                     crtc->base.base.id, crtc->base.name);
2501                         return -EINVAL;
2502                 }
2503
2504                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
2505                     intel_is_dual_link_lvds(i915)) {
2506                         drm_dbg_kms(&i915->drm,
2507                                     "[CRTC:%d:%s] Odd pipe source width not supported with dual link LVDS\n",
2508                                     crtc->base.base.id, crtc->base.name);
2509                         return -EINVAL;
2510                 }
2511         }
2512
2513         return 0;
2514 }
2515
2516 static int intel_crtc_compute_pipe_mode(struct intel_crtc_state *crtc_state)
2517 {
2518         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2519         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2520         struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
2521         struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
2522         int clock_limit = i915->max_dotclk_freq;
2523
2524         /*
2525          * Start with the adjusted_mode crtc timings, which
2526          * have been filled with the transcoder timings.
2527          */
2528         drm_mode_copy(pipe_mode, adjusted_mode);
2529
2530         /* Expand MSO per-segment transcoder timings to full */
2531         intel_splitter_adjust_timings(crtc_state, pipe_mode);
2532
2533         /* Derive per-pipe timings in case bigjoiner is used */
2534         intel_bigjoiner_adjust_timings(crtc_state, pipe_mode);
2535         intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
2536
2537         if (DISPLAY_VER(i915) < 4) {
2538                 clock_limit = i915->display.cdclk.max_cdclk_freq * 9 / 10;
2539
2540                 /*
2541                  * Enable double wide mode when the dot clock
2542                  * is > 90% of the (display) core speed.
2543                  */
2544                 if (intel_crtc_supports_double_wide(crtc) &&
2545                     pipe_mode->crtc_clock > clock_limit) {
2546                         clock_limit = i915->max_dotclk_freq;
2547                         crtc_state->double_wide = true;
2548                 }
2549         }
2550
2551         if (pipe_mode->crtc_clock > clock_limit) {
2552                 drm_dbg_kms(&i915->drm,
2553                             "[CRTC:%d:%s] requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
2554                             crtc->base.base.id, crtc->base.name,
2555                             pipe_mode->crtc_clock, clock_limit,
2556                             str_yes_no(crtc_state->double_wide));
2557                 return -EINVAL;
2558         }
2559
2560         return 0;
2561 }
2562
2563 static int intel_crtc_compute_config(struct intel_atomic_state *state,
2564                                      struct intel_crtc *crtc)
2565 {
2566         struct intel_crtc_state *crtc_state =
2567                 intel_atomic_get_new_crtc_state(state, crtc);
2568         int ret;
2569
2570         ret = intel_dpll_crtc_compute_clock(state, crtc);
2571         if (ret)
2572                 return ret;
2573
2574         ret = intel_crtc_compute_pipe_src(crtc_state);
2575         if (ret)
2576                 return ret;
2577
2578         ret = intel_crtc_compute_pipe_mode(crtc_state);
2579         if (ret)
2580                 return ret;
2581
2582         intel_crtc_compute_pixel_rate(crtc_state);
2583
2584         if (crtc_state->has_pch_encoder)
2585                 return ilk_fdi_compute_config(crtc, crtc_state);
2586
2587         return 0;
2588 }
2589
2590 static void
2591 intel_reduce_m_n_ratio(u32 *num, u32 *den)
2592 {
2593         while (*num > DATA_LINK_M_N_MASK ||
2594                *den > DATA_LINK_M_N_MASK) {
2595                 *num >>= 1;
2596                 *den >>= 1;
2597         }
2598 }
2599
2600 static void compute_m_n(u32 *ret_m, u32 *ret_n,
2601                         u32 m, u32 n, u32 constant_n)
2602 {
2603         if (constant_n)
2604                 *ret_n = constant_n;
2605         else
2606                 *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
2607
2608         *ret_m = div_u64(mul_u32_u32(m, *ret_n), n);
2609         intel_reduce_m_n_ratio(ret_m, ret_n);
2610 }
2611
2612 void
2613 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
2614                        int pixel_clock, int link_clock,
2615                        struct intel_link_m_n *m_n,
2616                        bool fec_enable)
2617 {
2618         u32 data_clock = bits_per_pixel * pixel_clock;
2619
2620         if (fec_enable)
2621                 data_clock = intel_dp_mode_to_fec_clock(data_clock);
2622
2623         /*
2624          * Windows/BIOS uses fixed M/N values always. Follow suit.
2625          *
2626          * Also several DP dongles in particular seem to be fussy
2627          * about too large link M/N values. Presumably the 20bit
2628          * value used by Windows/BIOS is acceptable to everyone.
2629          */
2630         m_n->tu = 64;
2631         compute_m_n(&m_n->data_m, &m_n->data_n,
2632                     data_clock, link_clock * nlanes * 8,
2633                     0x8000000);
2634
2635         compute_m_n(&m_n->link_m, &m_n->link_n,
2636                     pixel_clock, link_clock,
2637                     0x80000);
2638 }
2639
2640 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
2641 {
2642         /*
2643          * There may be no VBT; and if the BIOS enabled SSC we can
2644          * just keep using it to avoid unnecessary flicker.  Whereas if the
2645          * BIOS isn't using it, don't assume it will work even if the VBT
2646          * indicates as much.
2647          */
2648         if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)) {
2649                 bool bios_lvds_use_ssc = intel_de_read(dev_priv,
2650                                                        PCH_DREF_CONTROL) &
2651                         DREF_SSC1_ENABLE;
2652
2653                 if (dev_priv->display.vbt.lvds_use_ssc != bios_lvds_use_ssc) {
2654                         drm_dbg_kms(&dev_priv->drm,
2655                                     "SSC %s by BIOS, overriding VBT which says %s\n",
2656                                     str_enabled_disabled(bios_lvds_use_ssc),
2657                                     str_enabled_disabled(dev_priv->display.vbt.lvds_use_ssc));
2658                         dev_priv->display.vbt.lvds_use_ssc = bios_lvds_use_ssc;
2659                 }
2660         }
2661 }
2662
2663 void intel_zero_m_n(struct intel_link_m_n *m_n)
2664 {
2665         /* corresponds to 0 register value */
2666         memset(m_n, 0, sizeof(*m_n));
2667         m_n->tu = 1;
2668 }
2669
2670 void intel_set_m_n(struct drm_i915_private *i915,
2671                    const struct intel_link_m_n *m_n,
2672                    i915_reg_t data_m_reg, i915_reg_t data_n_reg,
2673                    i915_reg_t link_m_reg, i915_reg_t link_n_reg)
2674 {
2675         intel_de_write(i915, data_m_reg, TU_SIZE(m_n->tu) | m_n->data_m);
2676         intel_de_write(i915, data_n_reg, m_n->data_n);
2677         intel_de_write(i915, link_m_reg, m_n->link_m);
2678         /*
2679          * On BDW+ writing LINK_N arms the double buffered update
2680          * of all the M/N registers, so it must be written last.
2681          */
2682         intel_de_write(i915, link_n_reg, m_n->link_n);
2683 }
2684
2685 bool intel_cpu_transcoder_has_m2_n2(struct drm_i915_private *dev_priv,
2686                                     enum transcoder transcoder)
2687 {
2688         if (IS_HASWELL(dev_priv))
2689                 return transcoder == TRANSCODER_EDP;
2690
2691         return IS_DISPLAY_VER(dev_priv, 5, 7) || IS_CHERRYVIEW(dev_priv);
2692 }
2693
2694 void intel_cpu_transcoder_set_m1_n1(struct intel_crtc *crtc,
2695                                     enum transcoder transcoder,
2696                                     const struct intel_link_m_n *m_n)
2697 {
2698         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2699         enum pipe pipe = crtc->pipe;
2700
2701         if (DISPLAY_VER(dev_priv) >= 5)
2702                 intel_set_m_n(dev_priv, m_n,
2703                               PIPE_DATA_M1(transcoder), PIPE_DATA_N1(transcoder),
2704                               PIPE_LINK_M1(transcoder), PIPE_LINK_N1(transcoder));
2705         else
2706                 intel_set_m_n(dev_priv, m_n,
2707                               PIPE_DATA_M_G4X(pipe), PIPE_DATA_N_G4X(pipe),
2708                               PIPE_LINK_M_G4X(pipe), PIPE_LINK_N_G4X(pipe));
2709 }
2710
2711 void intel_cpu_transcoder_set_m2_n2(struct intel_crtc *crtc,
2712                                     enum transcoder transcoder,
2713                                     const struct intel_link_m_n *m_n)
2714 {
2715         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2716
2717         if (!intel_cpu_transcoder_has_m2_n2(dev_priv, transcoder))
2718                 return;
2719
2720         intel_set_m_n(dev_priv, m_n,
2721                       PIPE_DATA_M2(transcoder), PIPE_DATA_N2(transcoder),
2722                       PIPE_LINK_M2(transcoder), PIPE_LINK_N2(transcoder));
2723 }
2724
2725 static void intel_set_transcoder_timings(const struct intel_crtc_state *crtc_state)
2726 {
2727         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2728         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2729         enum pipe pipe = crtc->pipe;
2730         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2731         const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
2732         u32 crtc_vdisplay, crtc_vtotal, crtc_vblank_start, crtc_vblank_end;
2733         int vsyncshift = 0;
2734
2735         /* We need to be careful not to changed the adjusted mode, for otherwise
2736          * the hw state checker will get angry at the mismatch. */
2737         crtc_vdisplay = adjusted_mode->crtc_vdisplay;
2738         crtc_vtotal = adjusted_mode->crtc_vtotal;
2739         crtc_vblank_start = adjusted_mode->crtc_vblank_start;
2740         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
2741
2742         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
2743                 /* the chip adds 2 halflines automatically */
2744                 crtc_vtotal -= 1;
2745                 crtc_vblank_end -= 1;
2746
2747                 if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
2748                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
2749                 else
2750                         vsyncshift = adjusted_mode->crtc_hsync_start -
2751                                 adjusted_mode->crtc_htotal / 2;
2752                 if (vsyncshift < 0)
2753                         vsyncshift += adjusted_mode->crtc_htotal;
2754         }
2755
2756         /*
2757          * VBLANK_START no longer works on ADL+, instead we must use
2758          * TRANS_SET_CONTEXT_LATENCY to configure the pipe vblank start.
2759          */
2760         if (DISPLAY_VER(dev_priv) >= 13) {
2761                 intel_de_write(dev_priv, TRANS_SET_CONTEXT_LATENCY(cpu_transcoder),
2762                                crtc_vblank_start - crtc_vdisplay);
2763
2764                 /*
2765                  * VBLANK_START not used by hw, just clear it
2766                  * to make it stand out in register dumps.
2767                  */
2768                 crtc_vblank_start = 1;
2769         }
2770
2771         if (DISPLAY_VER(dev_priv) > 3)
2772                 intel_de_write(dev_priv, TRANS_VSYNCSHIFT(cpu_transcoder),
2773                                vsyncshift);
2774
2775         intel_de_write(dev_priv, TRANS_HTOTAL(cpu_transcoder),
2776                        HACTIVE(adjusted_mode->crtc_hdisplay - 1) |
2777                        HTOTAL(adjusted_mode->crtc_htotal - 1));
2778         intel_de_write(dev_priv, TRANS_HBLANK(cpu_transcoder),
2779                        HBLANK_START(adjusted_mode->crtc_hblank_start - 1) |
2780                        HBLANK_END(adjusted_mode->crtc_hblank_end - 1));
2781         intel_de_write(dev_priv, TRANS_HSYNC(cpu_transcoder),
2782                        HSYNC_START(adjusted_mode->crtc_hsync_start - 1) |
2783                        HSYNC_END(adjusted_mode->crtc_hsync_end - 1));
2784
2785         intel_de_write(dev_priv, TRANS_VTOTAL(cpu_transcoder),
2786                        VACTIVE(crtc_vdisplay - 1) |
2787                        VTOTAL(crtc_vtotal - 1));
2788         intel_de_write(dev_priv, TRANS_VBLANK(cpu_transcoder),
2789                        VBLANK_START(crtc_vblank_start - 1) |
2790                        VBLANK_END(crtc_vblank_end - 1));
2791         intel_de_write(dev_priv, TRANS_VSYNC(cpu_transcoder),
2792                        VSYNC_START(adjusted_mode->crtc_vsync_start - 1) |
2793                        VSYNC_END(adjusted_mode->crtc_vsync_end - 1));
2794
2795         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
2796          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
2797          * documented on the DDI_FUNC_CTL register description, EDP Input Select
2798          * bits. */
2799         if (IS_HASWELL(dev_priv) && cpu_transcoder == TRANSCODER_EDP &&
2800             (pipe == PIPE_B || pipe == PIPE_C))
2801                 intel_de_write(dev_priv, TRANS_VTOTAL(pipe),
2802                                VACTIVE(crtc_vdisplay - 1) |
2803                                VTOTAL(crtc_vtotal - 1));
2804 }
2805
2806 static void intel_set_pipe_src_size(const struct intel_crtc_state *crtc_state)
2807 {
2808         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2809         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2810         int width = drm_rect_width(&crtc_state->pipe_src);
2811         int height = drm_rect_height(&crtc_state->pipe_src);
2812         enum pipe pipe = crtc->pipe;
2813
2814         /* pipesrc controls the size that is scaled from, which should
2815          * always be the user's requested size.
2816          */
2817         intel_de_write(dev_priv, PIPESRC(pipe),
2818                        PIPESRC_WIDTH(width - 1) | PIPESRC_HEIGHT(height - 1));
2819 }
2820
2821 static bool intel_pipe_is_interlaced(const struct intel_crtc_state *crtc_state)
2822 {
2823         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2824         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2825
2826         if (DISPLAY_VER(dev_priv) == 2)
2827                 return false;
2828
2829         if (DISPLAY_VER(dev_priv) >= 9 ||
2830             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2831                 return intel_de_read(dev_priv, TRANSCONF(cpu_transcoder)) & TRANSCONF_INTERLACE_MASK_HSW;
2832         else
2833                 return intel_de_read(dev_priv, TRANSCONF(cpu_transcoder)) & TRANSCONF_INTERLACE_MASK;
2834 }
2835
2836 static void intel_get_transcoder_timings(struct intel_crtc *crtc,
2837                                          struct intel_crtc_state *pipe_config)
2838 {
2839         struct drm_device *dev = crtc->base.dev;
2840         struct drm_i915_private *dev_priv = to_i915(dev);
2841         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
2842         struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
2843         u32 tmp;
2844
2845         tmp = intel_de_read(dev_priv, TRANS_HTOTAL(cpu_transcoder));
2846         adjusted_mode->crtc_hdisplay = REG_FIELD_GET(HACTIVE_MASK, tmp) + 1;
2847         adjusted_mode->crtc_htotal = REG_FIELD_GET(HTOTAL_MASK, tmp) + 1;
2848
2849         if (!transcoder_is_dsi(cpu_transcoder)) {
2850                 tmp = intel_de_read(dev_priv, TRANS_HBLANK(cpu_transcoder));
2851                 adjusted_mode->crtc_hblank_start = REG_FIELD_GET(HBLANK_START_MASK, tmp) + 1;
2852                 adjusted_mode->crtc_hblank_end = REG_FIELD_GET(HBLANK_END_MASK, tmp) + 1;
2853         }
2854
2855         tmp = intel_de_read(dev_priv, TRANS_HSYNC(cpu_transcoder));
2856         adjusted_mode->crtc_hsync_start = REG_FIELD_GET(HSYNC_START_MASK, tmp) + 1;
2857         adjusted_mode->crtc_hsync_end = REG_FIELD_GET(HSYNC_END_MASK, tmp) + 1;
2858
2859         tmp = intel_de_read(dev_priv, TRANS_VTOTAL(cpu_transcoder));
2860         adjusted_mode->crtc_vdisplay = REG_FIELD_GET(VACTIVE_MASK, tmp) + 1;
2861         adjusted_mode->crtc_vtotal = REG_FIELD_GET(VTOTAL_MASK, tmp) + 1;
2862
2863         /* FIXME TGL+ DSI transcoders have this! */
2864         if (!transcoder_is_dsi(cpu_transcoder)) {
2865                 tmp = intel_de_read(dev_priv, TRANS_VBLANK(cpu_transcoder));
2866                 adjusted_mode->crtc_vblank_start = REG_FIELD_GET(VBLANK_START_MASK, tmp) + 1;
2867                 adjusted_mode->crtc_vblank_end = REG_FIELD_GET(VBLANK_END_MASK, tmp) + 1;
2868         }
2869         tmp = intel_de_read(dev_priv, TRANS_VSYNC(cpu_transcoder));
2870         adjusted_mode->crtc_vsync_start = REG_FIELD_GET(VSYNC_START_MASK, tmp) + 1;
2871         adjusted_mode->crtc_vsync_end = REG_FIELD_GET(VSYNC_END_MASK, tmp) + 1;
2872
2873         if (intel_pipe_is_interlaced(pipe_config)) {
2874                 adjusted_mode->flags |= DRM_MODE_FLAG_INTERLACE;
2875                 adjusted_mode->crtc_vtotal += 1;
2876                 adjusted_mode->crtc_vblank_end += 1;
2877         }
2878
2879         if (DISPLAY_VER(dev_priv) >= 13 && !transcoder_is_dsi(cpu_transcoder))
2880                 adjusted_mode->crtc_vblank_start =
2881                         adjusted_mode->crtc_vdisplay +
2882                         intel_de_read(dev_priv, TRANS_SET_CONTEXT_LATENCY(cpu_transcoder));
2883 }
2884
2885 static void intel_bigjoiner_adjust_pipe_src(struct intel_crtc_state *crtc_state)
2886 {
2887         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2888         int num_pipes = intel_bigjoiner_num_pipes(crtc_state);
2889         enum pipe master_pipe, pipe = crtc->pipe;
2890         int width;
2891
2892         if (num_pipes < 2)
2893                 return;
2894
2895         master_pipe = bigjoiner_master_pipe(crtc_state);
2896         width = drm_rect_width(&crtc_state->pipe_src);
2897
2898         drm_rect_translate_to(&crtc_state->pipe_src,
2899                               (pipe - master_pipe) * width, 0);
2900 }
2901
2902 static void intel_get_pipe_src_size(struct intel_crtc *crtc,
2903                                     struct intel_crtc_state *pipe_config)
2904 {
2905         struct drm_device *dev = crtc->base.dev;
2906         struct drm_i915_private *dev_priv = to_i915(dev);
2907         u32 tmp;
2908
2909         tmp = intel_de_read(dev_priv, PIPESRC(crtc->pipe));
2910
2911         drm_rect_init(&pipe_config->pipe_src, 0, 0,
2912                       REG_FIELD_GET(PIPESRC_WIDTH_MASK, tmp) + 1,
2913                       REG_FIELD_GET(PIPESRC_HEIGHT_MASK, tmp) + 1);
2914
2915         intel_bigjoiner_adjust_pipe_src(pipe_config);
2916 }
2917
2918 void i9xx_set_pipeconf(const struct intel_crtc_state *crtc_state)
2919 {
2920         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2921         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2922         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
2923         u32 val = 0;
2924
2925         /*
2926          * - We keep both pipes enabled on 830
2927          * - During modeset the pipe is still disabled and must remain so
2928          * - During fastset the pipe is already enabled and must remain so
2929          */
2930         if (IS_I830(dev_priv) || !intel_crtc_needs_modeset(crtc_state))
2931                 val |= TRANSCONF_ENABLE;
2932
2933         if (crtc_state->double_wide)
2934                 val |= TRANSCONF_DOUBLE_WIDE;
2935
2936         /* only g4x and later have fancy bpc/dither controls */
2937         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
2938             IS_CHERRYVIEW(dev_priv)) {
2939                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
2940                 if (crtc_state->dither && crtc_state->pipe_bpp != 30)
2941                         val |= TRANSCONF_DITHER_EN |
2942                                 TRANSCONF_DITHER_TYPE_SP;
2943
2944                 switch (crtc_state->pipe_bpp) {
2945                 default:
2946                         /* Case prevented by intel_choose_pipe_bpp_dither. */
2947                         MISSING_CASE(crtc_state->pipe_bpp);
2948                         fallthrough;
2949                 case 18:
2950                         val |= TRANSCONF_BPC_6;
2951                         break;
2952                 case 24:
2953                         val |= TRANSCONF_BPC_8;
2954                         break;
2955                 case 30:
2956                         val |= TRANSCONF_BPC_10;
2957                         break;
2958                 }
2959         }
2960
2961         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
2962                 if (DISPLAY_VER(dev_priv) < 4 ||
2963                     intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
2964                         val |= TRANSCONF_INTERLACE_W_FIELD_INDICATION;
2965                 else
2966                         val |= TRANSCONF_INTERLACE_W_SYNC_SHIFT;
2967         } else {
2968                 val |= TRANSCONF_INTERLACE_PROGRESSIVE;
2969         }
2970
2971         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2972              crtc_state->limited_color_range)
2973                 val |= TRANSCONF_COLOR_RANGE_SELECT;
2974
2975         val |= TRANSCONF_GAMMA_MODE(crtc_state->gamma_mode);
2976
2977         val |= TRANSCONF_FRAME_START_DELAY(crtc_state->framestart_delay - 1);
2978
2979         intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
2980         intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
2981 }
2982
2983 static bool i9xx_has_pfit(struct drm_i915_private *dev_priv)
2984 {
2985         if (IS_I830(dev_priv))
2986                 return false;
2987
2988         return DISPLAY_VER(dev_priv) >= 4 ||
2989                 IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
2990 }
2991
2992 static void i9xx_get_pfit_config(struct intel_crtc_state *crtc_state)
2993 {
2994         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2995         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2996         u32 tmp;
2997
2998         if (!i9xx_has_pfit(dev_priv))
2999                 return;
3000
3001         tmp = intel_de_read(dev_priv, PFIT_CONTROL);
3002         if (!(tmp & PFIT_ENABLE))
3003                 return;
3004
3005         /* Check whether the pfit is attached to our pipe. */
3006         if (DISPLAY_VER(dev_priv) < 4) {
3007                 if (crtc->pipe != PIPE_B)
3008                         return;
3009         } else {
3010                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
3011                         return;
3012         }
3013
3014         crtc_state->gmch_pfit.control = tmp;
3015         crtc_state->gmch_pfit.pgm_ratios =
3016                 intel_de_read(dev_priv, PFIT_PGM_RATIOS);
3017 }
3018
3019 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
3020                                struct intel_crtc_state *pipe_config)
3021 {
3022         struct drm_device *dev = crtc->base.dev;
3023         struct drm_i915_private *dev_priv = to_i915(dev);
3024         enum pipe pipe = crtc->pipe;
3025         struct dpll clock;
3026         u32 mdiv;
3027         int refclk = 100000;
3028
3029         /* In case of DSI, DPLL will not be used */
3030         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
3031                 return;
3032
3033         vlv_dpio_get(dev_priv);
3034         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
3035         vlv_dpio_put(dev_priv);
3036
3037         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
3038         clock.m2 = mdiv & DPIO_M2DIV_MASK;
3039         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
3040         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
3041         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
3042
3043         pipe_config->port_clock = vlv_calc_dpll_params(refclk, &clock);
3044 }
3045
3046 static void chv_crtc_clock_get(struct intel_crtc *crtc,
3047                                struct intel_crtc_state *pipe_config)
3048 {
3049         struct drm_device *dev = crtc->base.dev;
3050         struct drm_i915_private *dev_priv = to_i915(dev);
3051         enum pipe pipe = crtc->pipe;
3052         enum dpio_channel port = vlv_pipe_to_channel(pipe);
3053         struct dpll clock;
3054         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
3055         int refclk = 100000;
3056
3057         /* In case of DSI, DPLL will not be used */
3058         if ((pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE) == 0)
3059                 return;
3060
3061         vlv_dpio_get(dev_priv);
3062         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
3063         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
3064         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
3065         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
3066         pll_dw3 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
3067         vlv_dpio_put(dev_priv);
3068
3069         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
3070         clock.m2 = (pll_dw0 & 0xff) << 22;
3071         if (pll_dw3 & DPIO_CHV_FRAC_DIV_EN)
3072                 clock.m2 |= pll_dw2 & 0x3fffff;
3073         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
3074         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
3075         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
3076
3077         pipe_config->port_clock = chv_calc_dpll_params(refclk, &clock);
3078 }
3079
3080 static enum intel_output_format
3081 bdw_get_pipe_misc_output_format(struct intel_crtc *crtc)
3082 {
3083         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3084         u32 tmp;
3085
3086         tmp = intel_de_read(dev_priv, PIPE_MISC(crtc->pipe));
3087
3088         if (tmp & PIPE_MISC_YUV420_ENABLE) {
3089                 /* We support 4:2:0 in full blend mode only */
3090                 drm_WARN_ON(&dev_priv->drm,
3091                             (tmp & PIPE_MISC_YUV420_MODE_FULL_BLEND) == 0);
3092
3093                 return INTEL_OUTPUT_FORMAT_YCBCR420;
3094         } else if (tmp & PIPE_MISC_OUTPUT_COLORSPACE_YUV) {
3095                 return INTEL_OUTPUT_FORMAT_YCBCR444;
3096         } else {
3097                 return INTEL_OUTPUT_FORMAT_RGB;
3098         }
3099 }
3100
3101 static void i9xx_get_pipe_color_config(struct intel_crtc_state *crtc_state)
3102 {
3103         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3104         struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3105         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3106         enum i9xx_plane_id i9xx_plane = plane->i9xx_plane;
3107         u32 tmp;
3108
3109         tmp = intel_de_read(dev_priv, DSPCNTR(i9xx_plane));
3110
3111         if (tmp & DISP_PIPE_GAMMA_ENABLE)
3112                 crtc_state->gamma_enable = true;
3113
3114         if (!HAS_GMCH(dev_priv) &&
3115             tmp & DISP_PIPE_CSC_ENABLE)
3116                 crtc_state->csc_enable = true;
3117 }
3118
3119 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
3120                                  struct intel_crtc_state *pipe_config)
3121 {
3122         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3123         enum intel_display_power_domain power_domain;
3124         intel_wakeref_t wakeref;
3125         u32 tmp;
3126         bool ret;
3127
3128         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
3129         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
3130         if (!wakeref)
3131                 return false;
3132
3133         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
3134         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
3135         pipe_config->shared_dpll = NULL;
3136
3137         ret = false;
3138
3139         tmp = intel_de_read(dev_priv, TRANSCONF(pipe_config->cpu_transcoder));
3140         if (!(tmp & TRANSCONF_ENABLE))
3141                 goto out;
3142
3143         if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
3144             IS_CHERRYVIEW(dev_priv)) {
3145                 switch (tmp & TRANSCONF_BPC_MASK) {
3146                 case TRANSCONF_BPC_6:
3147                         pipe_config->pipe_bpp = 18;
3148                         break;
3149                 case TRANSCONF_BPC_8:
3150                         pipe_config->pipe_bpp = 24;
3151                         break;
3152                 case TRANSCONF_BPC_10:
3153                         pipe_config->pipe_bpp = 30;
3154                         break;
3155                 default:
3156                         MISSING_CASE(tmp);
3157                         break;
3158                 }
3159         }
3160
3161         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
3162             (tmp & TRANSCONF_COLOR_RANGE_SELECT))
3163                 pipe_config->limited_color_range = true;
3164
3165         pipe_config->gamma_mode = REG_FIELD_GET(TRANSCONF_GAMMA_MODE_MASK_I9XX, tmp);
3166
3167         pipe_config->framestart_delay = REG_FIELD_GET(TRANSCONF_FRAME_START_DELAY_MASK, tmp) + 1;
3168
3169         if (IS_CHERRYVIEW(dev_priv))
3170                 pipe_config->cgm_mode = intel_de_read(dev_priv,
3171                                                       CGM_PIPE_MODE(crtc->pipe));
3172
3173         i9xx_get_pipe_color_config(pipe_config);
3174         intel_color_get_config(pipe_config);
3175
3176         if (DISPLAY_VER(dev_priv) < 4)
3177                 pipe_config->double_wide = tmp & TRANSCONF_DOUBLE_WIDE;
3178
3179         intel_get_transcoder_timings(crtc, pipe_config);
3180         intel_get_pipe_src_size(crtc, pipe_config);
3181
3182         i9xx_get_pfit_config(pipe_config);
3183
3184         if (DISPLAY_VER(dev_priv) >= 4) {
3185                 /* No way to read it out on pipes B and C */
3186                 if (IS_CHERRYVIEW(dev_priv) && crtc->pipe != PIPE_A)
3187                         tmp = dev_priv->display.state.chv_dpll_md[crtc->pipe];
3188                 else
3189                         tmp = intel_de_read(dev_priv, DPLL_MD(crtc->pipe));
3190                 pipe_config->pixel_multiplier =
3191                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
3192                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
3193                 pipe_config->dpll_hw_state.dpll_md = tmp;
3194         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
3195                    IS_G33(dev_priv) || IS_PINEVIEW(dev_priv)) {
3196                 tmp = intel_de_read(dev_priv, DPLL(crtc->pipe));
3197                 pipe_config->pixel_multiplier =
3198                         ((tmp & SDVO_MULTIPLIER_MASK)
3199                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
3200         } else {
3201                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
3202                  * port and will be fixed up in the encoder->get_config
3203                  * function. */
3204                 pipe_config->pixel_multiplier = 1;
3205         }
3206         pipe_config->dpll_hw_state.dpll = intel_de_read(dev_priv,
3207                                                         DPLL(crtc->pipe));
3208         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
3209                 pipe_config->dpll_hw_state.fp0 = intel_de_read(dev_priv,
3210                                                                FP0(crtc->pipe));
3211                 pipe_config->dpll_hw_state.fp1 = intel_de_read(dev_priv,
3212                                                                FP1(crtc->pipe));
3213         } else {
3214                 /* Mask out read-only status bits. */
3215                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
3216                                                      DPLL_PORTC_READY_MASK |
3217                                                      DPLL_PORTB_READY_MASK);
3218         }
3219
3220         if (IS_CHERRYVIEW(dev_priv))
3221                 chv_crtc_clock_get(crtc, pipe_config);
3222         else if (IS_VALLEYVIEW(dev_priv))
3223                 vlv_crtc_clock_get(crtc, pipe_config);
3224         else
3225                 i9xx_crtc_clock_get(crtc, pipe_config);
3226
3227         /*
3228          * Normally the dotclock is filled in by the encoder .get_config()
3229          * but in case the pipe is enabled w/o any ports we need a sane
3230          * default.
3231          */
3232         pipe_config->hw.adjusted_mode.crtc_clock =
3233                 pipe_config->port_clock / pipe_config->pixel_multiplier;
3234
3235         ret = true;
3236
3237 out:
3238         intel_display_power_put(dev_priv, power_domain, wakeref);
3239
3240         return ret;
3241 }
3242
3243 void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state)
3244 {
3245         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3246         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3247         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
3248         u32 val = 0;
3249
3250         /*
3251          * - During modeset the pipe is still disabled and must remain so
3252          * - During fastset the pipe is already enabled and must remain so
3253          */
3254         if (!intel_crtc_needs_modeset(crtc_state))
3255                 val |= TRANSCONF_ENABLE;
3256
3257         switch (crtc_state->pipe_bpp) {
3258         default:
3259                 /* Case prevented by intel_choose_pipe_bpp_dither. */
3260                 MISSING_CASE(crtc_state->pipe_bpp);
3261                 fallthrough;
3262         case 18:
3263                 val |= TRANSCONF_BPC_6;
3264                 break;
3265         case 24:
3266                 val |= TRANSCONF_BPC_8;
3267                 break;
3268         case 30:
3269                 val |= TRANSCONF_BPC_10;
3270                 break;
3271         case 36:
3272                 val |= TRANSCONF_BPC_12;
3273                 break;
3274         }
3275
3276         if (crtc_state->dither)
3277                 val |= TRANSCONF_DITHER_EN | TRANSCONF_DITHER_TYPE_SP;
3278
3279         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3280                 val |= TRANSCONF_INTERLACE_IF_ID_ILK;
3281         else
3282                 val |= TRANSCONF_INTERLACE_PF_PD_ILK;
3283
3284         /*
3285          * This would end up with an odd purple hue over
3286          * the entire display. Make sure we don't do it.
3287          */
3288         drm_WARN_ON(&dev_priv->drm, crtc_state->limited_color_range &&
3289                     crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB);
3290
3291         if (crtc_state->limited_color_range &&
3292             !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO))
3293                 val |= TRANSCONF_COLOR_RANGE_SELECT;
3294
3295         if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
3296                 val |= TRANSCONF_OUTPUT_COLORSPACE_YUV709;
3297
3298         val |= TRANSCONF_GAMMA_MODE(crtc_state->gamma_mode);
3299
3300         val |= TRANSCONF_FRAME_START_DELAY(crtc_state->framestart_delay - 1);
3301         val |= TRANSCONF_MSA_TIMING_DELAY(crtc_state->msa_timing_delay);
3302
3303         intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
3304         intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
3305 }
3306
3307 static void hsw_set_transconf(const struct intel_crtc_state *crtc_state)
3308 {
3309         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3310         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3311         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
3312         u32 val = 0;
3313
3314         /*
3315          * - During modeset the pipe is still disabled and must remain so
3316          * - During fastset the pipe is already enabled and must remain so
3317          */
3318         if (!intel_crtc_needs_modeset(crtc_state))
3319                 val |= TRANSCONF_ENABLE;
3320
3321         if (IS_HASWELL(dev_priv) && crtc_state->dither)
3322                 val |= TRANSCONF_DITHER_EN | TRANSCONF_DITHER_TYPE_SP;
3323
3324         if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
3325                 val |= TRANSCONF_INTERLACE_IF_ID_ILK;
3326         else
3327                 val |= TRANSCONF_INTERLACE_PF_PD_ILK;
3328
3329         if (IS_HASWELL(dev_priv) &&
3330             crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)
3331                 val |= TRANSCONF_OUTPUT_COLORSPACE_YUV_HSW;
3332
3333         intel_de_write(dev_priv, TRANSCONF(cpu_transcoder), val);
3334         intel_de_posting_read(dev_priv, TRANSCONF(cpu_transcoder));
3335 }
3336
3337 static void bdw_set_pipe_misc(const struct intel_crtc_state *crtc_state)
3338 {
3339         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3340         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3341         u32 val = 0;
3342
3343         switch (crtc_state->pipe_bpp) {
3344         case 18:
3345                 val |= PIPE_MISC_BPC_6;
3346                 break;
3347         case 24:
3348                 val |= PIPE_MISC_BPC_8;
3349                 break;
3350         case 30:
3351                 val |= PIPE_MISC_BPC_10;
3352                 break;
3353         case 36:
3354                 /* Port output 12BPC defined for ADLP+ */
3355                 if (DISPLAY_VER(dev_priv) > 12)
3356                         val |= PIPE_MISC_BPC_12_ADLP;
3357                 break;
3358         default:
3359                 MISSING_CASE(crtc_state->pipe_bpp);
3360                 break;
3361         }
3362
3363         if (crtc_state->dither)
3364                 val |= PIPE_MISC_DITHER_ENABLE | PIPE_MISC_DITHER_TYPE_SP;
3365
3366         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
3367             crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR444)
3368                 val |= PIPE_MISC_OUTPUT_COLORSPACE_YUV;
3369
3370         if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420)
3371                 val |= PIPE_MISC_YUV420_ENABLE |
3372                         PIPE_MISC_YUV420_MODE_FULL_BLEND;
3373
3374         if (DISPLAY_VER(dev_priv) >= 11 && is_hdr_mode(crtc_state))
3375                 val |= PIPE_MISC_HDR_MODE_PRECISION;
3376
3377         if (DISPLAY_VER(dev_priv) >= 12)
3378                 val |= PIPE_MISC_PIXEL_ROUNDING_TRUNC;
3379
3380         intel_de_write(dev_priv, PIPE_MISC(crtc->pipe), val);
3381 }
3382
3383 int bdw_get_pipe_misc_bpp(struct intel_crtc *crtc)
3384 {
3385         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3386         u32 tmp;
3387
3388         tmp = intel_de_read(dev_priv, PIPE_MISC(crtc->pipe));
3389
3390         switch (tmp & PIPE_MISC_BPC_MASK) {
3391         case PIPE_MISC_BPC_6:
3392                 return 18;
3393         case PIPE_MISC_BPC_8:
3394                 return 24;
3395         case PIPE_MISC_BPC_10:
3396                 return 30;
3397         /*
3398          * PORT OUTPUT 12 BPC defined for ADLP+.
3399          *
3400          * TODO:
3401          * For previous platforms with DSI interface, bits 5:7
3402          * are used for storing pipe_bpp irrespective of dithering.
3403          * Since the value of 12 BPC is not defined for these bits
3404          * on older platforms, need to find a workaround for 12 BPC
3405          * MIPI DSI HW readout.
3406          */
3407         case PIPE_MISC_BPC_12_ADLP:
3408                 if (DISPLAY_VER(dev_priv) > 12)
3409                         return 36;
3410                 fallthrough;
3411         default:
3412                 MISSING_CASE(tmp);
3413                 return 0;
3414         }
3415 }
3416
3417 int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
3418 {
3419         /*
3420          * Account for spread spectrum to avoid
3421          * oversubscribing the link. Max center spread
3422          * is 2.5%; use 5% for safety's sake.
3423          */
3424         u32 bps = target_clock * bpp * 21 / 20;
3425         return DIV_ROUND_UP(bps, link_bw * 8);
3426 }
3427
3428 void intel_get_m_n(struct drm_i915_private *i915,
3429                    struct intel_link_m_n *m_n,
3430                    i915_reg_t data_m_reg, i915_reg_t data_n_reg,
3431                    i915_reg_t link_m_reg, i915_reg_t link_n_reg)
3432 {
3433         m_n->link_m = intel_de_read(i915, link_m_reg) & DATA_LINK_M_N_MASK;
3434         m_n->link_n = intel_de_read(i915, link_n_reg) & DATA_LINK_M_N_MASK;
3435         m_n->data_m = intel_de_read(i915, data_m_reg) & DATA_LINK_M_N_MASK;
3436         m_n->data_n = intel_de_read(i915, data_n_reg) & DATA_LINK_M_N_MASK;
3437         m_n->tu = REG_FIELD_GET(TU_SIZE_MASK, intel_de_read(i915, data_m_reg)) + 1;
3438 }
3439
3440 void intel_cpu_transcoder_get_m1_n1(struct intel_crtc *crtc,
3441                                     enum transcoder transcoder,
3442                                     struct intel_link_m_n *m_n)
3443 {
3444         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3445         enum pipe pipe = crtc->pipe;
3446
3447         if (DISPLAY_VER(dev_priv) >= 5)
3448                 intel_get_m_n(dev_priv, m_n,
3449                               PIPE_DATA_M1(transcoder), PIPE_DATA_N1(transcoder),
3450                               PIPE_LINK_M1(transcoder), PIPE_LINK_N1(transcoder));
3451         else
3452                 intel_get_m_n(dev_priv, m_n,
3453                               PIPE_DATA_M_G4X(pipe), PIPE_DATA_N_G4X(pipe),
3454                               PIPE_LINK_M_G4X(pipe), PIPE_LINK_N_G4X(pipe));
3455 }
3456
3457 void intel_cpu_transcoder_get_m2_n2(struct intel_crtc *crtc,
3458                                     enum transcoder transcoder,
3459                                     struct intel_link_m_n *m_n)
3460 {
3461         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3462
3463         if (!intel_cpu_transcoder_has_m2_n2(dev_priv, transcoder))
3464                 return;
3465
3466         intel_get_m_n(dev_priv, m_n,
3467                       PIPE_DATA_M2(transcoder), PIPE_DATA_N2(transcoder),
3468                       PIPE_LINK_M2(transcoder), PIPE_LINK_N2(transcoder));
3469 }
3470
3471 static void ilk_get_pfit_pos_size(struct intel_crtc_state *crtc_state,
3472                                   u32 pos, u32 size)
3473 {
3474         drm_rect_init(&crtc_state->pch_pfit.dst,
3475                       pos >> 16, pos & 0xffff,
3476                       size >> 16, size & 0xffff);
3477 }
3478
3479 static void skl_get_pfit_config(struct intel_crtc_state *crtc_state)
3480 {
3481         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3482         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3483         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
3484         int id = -1;
3485         int i;
3486
3487         /* find scaler attached to this pipe */
3488         for (i = 0; i < crtc->num_scalers; i++) {
3489                 u32 ctl, pos, size;
3490
3491                 ctl = intel_de_read(dev_priv, SKL_PS_CTRL(crtc->pipe, i));
3492                 if ((ctl & (PS_SCALER_EN | PS_PLANE_SEL_MASK)) != PS_SCALER_EN)
3493                         continue;
3494
3495                 id = i;
3496                 crtc_state->pch_pfit.enabled = true;
3497
3498                 pos = intel_de_read(dev_priv, SKL_PS_WIN_POS(crtc->pipe, i));
3499                 size = intel_de_read(dev_priv, SKL_PS_WIN_SZ(crtc->pipe, i));
3500
3501                 ilk_get_pfit_pos_size(crtc_state, pos, size);
3502
3503                 scaler_state->scalers[i].in_use = true;
3504                 break;
3505         }
3506
3507         scaler_state->scaler_id = id;
3508         if (id >= 0)
3509                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
3510         else
3511                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
3512 }
3513
3514 static void ilk_get_pfit_config(struct intel_crtc_state *crtc_state)
3515 {
3516         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3517         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3518         u32 ctl, pos, size;
3519
3520         ctl = intel_de_read(dev_priv, PF_CTL(crtc->pipe));
3521         if ((ctl & PF_ENABLE) == 0)
3522                 return;
3523
3524         crtc_state->pch_pfit.enabled = true;
3525
3526         pos = intel_de_read(dev_priv, PF_WIN_POS(crtc->pipe));
3527         size = intel_de_read(dev_priv, PF_WIN_SZ(crtc->pipe));
3528
3529         ilk_get_pfit_pos_size(crtc_state, pos, size);
3530
3531         /*
3532          * We currently do not free assignements of panel fitters on
3533          * ivb/hsw (since we don't use the higher upscaling modes which
3534          * differentiates them) so just WARN about this case for now.
3535          */
3536         drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) == 7 &&
3537                     (ctl & PF_PIPE_SEL_MASK_IVB) != PF_PIPE_SEL_IVB(crtc->pipe));
3538 }
3539
3540 static bool ilk_get_pipe_config(struct intel_crtc *crtc,
3541                                 struct intel_crtc_state *pipe_config)
3542 {
3543         struct drm_device *dev = crtc->base.dev;
3544         struct drm_i915_private *dev_priv = to_i915(dev);
3545         enum intel_display_power_domain power_domain;
3546         intel_wakeref_t wakeref;
3547         u32 tmp;
3548         bool ret;
3549
3550         power_domain = POWER_DOMAIN_PIPE(crtc->pipe);
3551         wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
3552         if (!wakeref)
3553                 return false;
3554
3555         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
3556         pipe_config->shared_dpll = NULL;
3557
3558         ret = false;
3559         tmp = intel_de_read(dev_priv, TRANSCONF(pipe_config->cpu_transcoder));
3560         if (!(tmp & TRANSCONF_ENABLE))
3561                 goto out;
3562
3563         switch (tmp & TRANSCONF_BPC_MASK) {
3564         case TRANSCONF_BPC_6:
3565                 pipe_config->pipe_bpp = 18;
3566                 break;
3567         case TRANSCONF_BPC_8:
3568                 pipe_config->pipe_bpp = 24;
3569                 break;
3570         case TRANSCONF_BPC_10:
3571                 pipe_config->pipe_bpp = 30;
3572                 break;
3573         case TRANSCONF_BPC_12:
3574                 pipe_config->pipe_bpp = 36;
3575                 break;
3576         default:
3577                 break;
3578         }
3579
3580         if (tmp & TRANSCONF_COLOR_RANGE_SELECT)
3581                 pipe_config->limited_color_range = true;
3582
3583         switch (tmp & TRANSCONF_OUTPUT_COLORSPACE_MASK) {
3584         case TRANSCONF_OUTPUT_COLORSPACE_YUV601:
3585         case TRANSCONF_OUTPUT_COLORSPACE_YUV709:
3586                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
3587                 break;
3588         default:
3589                 pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
3590                 break;
3591         }
3592
3593         pipe_config->gamma_mode = REG_FIELD_GET(TRANSCONF_GAMMA_MODE_MASK_ILK, tmp);
3594
3595         pipe_config->framestart_delay = REG_FIELD_GET(TRANSCONF_FRAME_START_DELAY_MASK, tmp) + 1;
3596
3597         pipe_config->msa_timing_delay = REG_FIELD_GET(TRANSCONF_MSA_TIMING_DELAY_MASK, tmp);
3598
3599         pipe_config->csc_mode = intel_de_read(dev_priv,
3600                                               PIPE_CSC_MODE(crtc->pipe));
3601
3602         i9xx_get_pipe_color_config(pipe_config);
3603         intel_color_get_config(pipe_config);
3604
3605         pipe_config->pixel_multiplier = 1;
3606
3607         ilk_pch_get_config(pipe_config);
3608
3609         intel_get_transcoder_timings(crtc, pipe_config);
3610         intel_get_pipe_src_size(crtc, pipe_config);
3611
3612         ilk_get_pfit_config(pipe_config);
3613
3614         ret = true;
3615
3616 out:
3617         intel_display_power_put(dev_priv, power_domain, wakeref);
3618
3619         return ret;
3620 }
3621
3622 static u8 bigjoiner_pipes(struct drm_i915_private *i915)
3623 {
3624         u8 pipes;
3625
3626         if (DISPLAY_VER(i915) >= 12)
3627                 pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D);
3628         else if (DISPLAY_VER(i915) >= 11)
3629                 pipes = BIT(PIPE_B) | BIT(PIPE_C);
3630         else
3631                 pipes = 0;
3632
3633         return pipes & RUNTIME_INFO(i915)->pipe_mask;
3634 }
3635
3636 static bool transcoder_ddi_func_is_enabled(struct drm_i915_private *dev_priv,
3637                                            enum transcoder cpu_transcoder)
3638 {
3639         enum intel_display_power_domain power_domain;
3640         intel_wakeref_t wakeref;
3641         u32 tmp = 0;
3642
3643         power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
3644
3645         with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref)
3646                 tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder));
3647
3648         return tmp & TRANS_DDI_FUNC_ENABLE;
3649 }
3650
3651 static void enabled_bigjoiner_pipes(struct drm_i915_private *dev_priv,
3652                                     u8 *master_pipes, u8 *slave_pipes)
3653 {
3654         struct intel_crtc *crtc;
3655
3656         *master_pipes = 0;
3657         *slave_pipes = 0;
3658
3659         for_each_intel_crtc_in_pipe_mask(&dev_priv->drm, crtc,
3660                                          bigjoiner_pipes(dev_priv)) {
3661                 enum intel_display_power_domain power_domain;
3662                 enum pipe pipe = crtc->pipe;
3663                 intel_wakeref_t wakeref;
3664
3665                 power_domain = intel_dsc_power_domain(crtc, (enum transcoder) pipe);
3666                 with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref) {
3667                         u32 tmp = intel_de_read(dev_priv, ICL_PIPE_DSS_CTL1(pipe));
3668
3669                         if (!(tmp & BIG_JOINER_ENABLE))
3670                                 continue;
3671
3672                         if (tmp & MASTER_BIG_JOINER_ENABLE)
3673                                 *master_pipes |= BIT(pipe);
3674                         else
3675                                 *slave_pipes |= BIT(pipe);
3676                 }
3677
3678                 if (DISPLAY_VER(dev_priv) < 13)
3679                         continue;
3680
3681                 power_domain = POWER_DOMAIN_PIPE(pipe);
3682                 with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref) {
3683                         u32 tmp = intel_de_read(dev_priv, ICL_PIPE_DSS_CTL1(pipe));
3684
3685                         if (tmp & UNCOMPRESSED_JOINER_MASTER)
3686                                 *master_pipes |= BIT(pipe);
3687                         if (tmp & UNCOMPRESSED_JOINER_SLAVE)
3688                                 *slave_pipes |= BIT(pipe);
3689                 }
3690         }
3691
3692         /* Bigjoiner pipes should always be consecutive master and slave */
3693         drm_WARN(&dev_priv->drm, *slave_pipes != *master_pipes << 1,
3694                  "Bigjoiner misconfigured (master pipes 0x%x, slave pipes 0x%x)\n",
3695                  *master_pipes, *slave_pipes);
3696 }
3697
3698 static enum pipe get_bigjoiner_master_pipe(enum pipe pipe, u8 master_pipes, u8 slave_pipes)
3699 {
3700         if ((slave_pipes & BIT(pipe)) == 0)
3701                 return pipe;
3702
3703         /* ignore everything above our pipe */
3704         master_pipes &= ~GENMASK(7, pipe);
3705
3706         /* highest remaining bit should be our master pipe */
3707         return fls(master_pipes) - 1;
3708 }
3709
3710 static u8 get_bigjoiner_slave_pipes(enum pipe pipe, u8 master_pipes, u8 slave_pipes)
3711 {
3712         enum pipe master_pipe, next_master_pipe;
3713
3714         master_pipe = get_bigjoiner_master_pipe(pipe, master_pipes, slave_pipes);
3715
3716         if ((master_pipes & BIT(master_pipe)) == 0)
3717                 return 0;
3718
3719         /* ignore our master pipe and everything below it */
3720         master_pipes &= ~GENMASK(master_pipe, 0);
3721         /* make sure a high bit is set for the ffs() */
3722         master_pipes |= BIT(7);
3723         /* lowest remaining bit should be the next master pipe */
3724         next_master_pipe = ffs(master_pipes) - 1;
3725
3726         return slave_pipes & GENMASK(next_master_pipe - 1, master_pipe);
3727 }
3728
3729 static u8 hsw_panel_transcoders(struct drm_i915_private *i915)
3730 {
3731         u8 panel_transcoder_mask = BIT(TRANSCODER_EDP);
3732
3733         if (DISPLAY_VER(i915) >= 11)
3734                 panel_transcoder_mask |= BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1);
3735
3736         return panel_transcoder_mask;
3737 }
3738
3739 static u8 hsw_enabled_transcoders(struct intel_crtc *crtc)
3740 {
3741         struct drm_device *dev = crtc->base.dev;
3742         struct drm_i915_private *dev_priv = to_i915(dev);
3743         u8 panel_transcoder_mask = hsw_panel_transcoders(dev_priv);
3744         enum transcoder cpu_transcoder;
3745         u8 master_pipes, slave_pipes;
3746         u8 enabled_transcoders = 0;
3747
3748         /*
3749          * XXX: Do intel_display_power_get_if_enabled before reading this (for
3750          * consistency and less surprising code; it's in always on power).
3751          */
3752         for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder,
3753                                        panel_transcoder_mask) {
3754                 enum intel_display_power_domain power_domain;
3755                 intel_wakeref_t wakeref;
3756                 enum pipe trans_pipe;
3757                 u32 tmp = 0;
3758
3759                 power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder);
3760                 with_intel_display_power_if_enabled(dev_priv, power_domain, wakeref)
3761                         tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder));
3762
3763                 if (!(tmp & TRANS_DDI_FUNC_ENABLE))
3764                         continue;
3765
3766                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
3767                 default:
3768                         drm_WARN(dev, 1,
3769                                  "unknown pipe linked to transcoder %s\n",
3770                                  transcoder_name(cpu_transcoder));
3771                         fallthrough;
3772                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
3773                 case TRANS_DDI_EDP_INPUT_A_ON:
3774                         trans_pipe = PIPE_A;
3775                         break;
3776                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
3777                         trans_pipe = PIPE_B;
3778                         break;
3779                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
3780                         trans_pipe = PIPE_C;
3781                         break;
3782                 case TRANS_DDI_EDP_INPUT_D_ONOFF:
3783                         trans_pipe = PIPE_D;
3784                         break;
3785                 }
3786
3787                 if (trans_pipe == crtc->pipe)
3788                         enabled_transcoders |= BIT(cpu_transcoder);
3789         }
3790
3791         /* single pipe or bigjoiner master */
3792         cpu_transcoder = (enum transcoder) crtc->pipe;
3793         if (transcoder_ddi_func_is_enabled(dev_priv, cpu_transcoder))
3794                 enabled_transcoders |= BIT(cpu_transcoder);
3795
3796         /* bigjoiner slave -> consider the master pipe's transcoder as well */
3797         enabled_bigjoiner_pipes(dev_priv, &master_pipes, &slave_pipes);
3798         if (slave_pipes & BIT(crtc->pipe)) {
3799                 cpu_transcoder = (enum transcoder)
3800                         get_bigjoiner_master_pipe(crtc->pipe, master_pipes, slave_pipes);
3801                 if (transcoder_ddi_func_is_enabled(dev_priv, cpu_transcoder))
3802                         enabled_transcoders |= BIT(cpu_transcoder);
3803         }
3804
3805         return enabled_transcoders;
3806 }
3807
3808 static bool has_edp_transcoders(u8 enabled_transcoders)
3809 {
3810         return enabled_transcoders & BIT(TRANSCODER_EDP);
3811 }
3812
3813 static bool has_dsi_transcoders(u8 enabled_transcoders)
3814 {
3815         return enabled_transcoders & (BIT(TRANSCODER_DSI_0) |
3816                                       BIT(TRANSCODER_DSI_1));
3817 }
3818
3819 static bool has_pipe_transcoders(u8 enabled_transcoders)
3820 {
3821         return enabled_transcoders & ~(BIT(TRANSCODER_EDP) |
3822                                        BIT(TRANSCODER_DSI_0) |
3823                                        BIT(TRANSCODER_DSI_1));
3824 }
3825
3826 static void assert_enabled_transcoders(struct drm_i915_private *i915,
3827                                        u8 enabled_transcoders)
3828 {
3829         /* Only one type of transcoder please */
3830         drm_WARN_ON(&i915->drm,
3831                     has_edp_transcoders(enabled_transcoders) +
3832                     has_dsi_transcoders(enabled_transcoders) +
3833                     has_pipe_transcoders(enabled_transcoders) > 1);
3834
3835         /* Only DSI transcoders can be ganged */
3836         drm_WARN_ON(&i915->drm,
3837                     !has_dsi_transcoders(enabled_transcoders) &&
3838                     !is_power_of_2(enabled_transcoders));
3839 }
3840
3841 static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
3842                                      struct intel_crtc_state *pipe_config,
3843                                      struct intel_display_power_domain_set *power_domain_set)
3844 {
3845         struct drm_device *dev = crtc->base.dev;
3846         struct drm_i915_private *dev_priv = to_i915(dev);
3847         unsigned long enabled_transcoders;
3848         u32 tmp;
3849
3850         enabled_transcoders = hsw_enabled_transcoders(crtc);
3851         if (!enabled_transcoders)
3852                 return false;
3853
3854         assert_enabled_transcoders(dev_priv, enabled_transcoders);
3855
3856         /*
3857          * With the exception of DSI we should only ever have
3858          * a single enabled transcoder. With DSI let's just
3859          * pick the first one.
3860          */
3861         pipe_config->cpu_transcoder = ffs(enabled_transcoders) - 1;
3862
3863         if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
3864                                                        POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
3865                 return false;
3866
3867         if (hsw_panel_transcoders(dev_priv) & BIT(pipe_config->cpu_transcoder)) {
3868                 tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
3869
3870                 if ((tmp & TRANS_DDI_EDP_INPUT_MASK) == TRANS_DDI_EDP_INPUT_A_ONOFF)
3871                         pipe_config->pch_pfit.force_thru = true;
3872         }
3873
3874         tmp = intel_de_read(dev_priv, TRANSCONF(pipe_config->cpu_transcoder));
3875
3876         return tmp & TRANSCONF_ENABLE;
3877 }
3878
3879 static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
3880                                          struct intel_crtc_state *pipe_config,
3881                                          struct intel_display_power_domain_set *power_domain_set)
3882 {
3883         struct drm_device *dev = crtc->base.dev;
3884         struct drm_i915_private *dev_priv = to_i915(dev);
3885         enum transcoder cpu_transcoder;
3886         enum port port;
3887         u32 tmp;
3888
3889         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
3890                 if (port == PORT_A)
3891                         cpu_transcoder = TRANSCODER_DSI_A;
3892                 else
3893                         cpu_transcoder = TRANSCODER_DSI_C;
3894
3895                 if (!intel_display_power_get_in_set_if_enabled(dev_priv, power_domain_set,
3896                                                                POWER_DOMAIN_TRANSCODER(cpu_transcoder)))
3897                         continue;
3898
3899                 /*
3900                  * The PLL needs to be enabled with a valid divider
3901                  * configuration, otherwise accessing DSI registers will hang
3902                  * the machine. See BSpec North Display Engine
3903                  * registers/MIPI[BXT]. We can break out here early, since we
3904                  * need the same DSI PLL to be enabled for both DSI ports.
3905                  */
3906                 if (!bxt_dsi_pll_is_enabled(dev_priv))
3907                         break;
3908
3909                 /* XXX: this works for video mode only */
3910                 tmp = intel_de_read(dev_priv, BXT_MIPI_PORT_CTRL(port));
3911                 if (!(tmp & DPI_ENABLE))
3912                         continue;
3913
3914                 tmp = intel_de_read(dev_priv, MIPI_CTRL(port));
3915                 if ((tmp & BXT_PIPE_SELECT_MASK) != BXT_PIPE_SELECT(crtc->pipe))
3916                         continue;
3917
3918                 pipe_config->cpu_transcoder = cpu_transcoder;
3919                 break;
3920         }
3921
3922         return transcoder_is_dsi(pipe_config->cpu_transcoder);
3923 }
3924
3925 static void intel_bigjoiner_get_config(struct intel_crtc_state *crtc_state)
3926 {
3927         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3928         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
3929         u8 master_pipes, slave_pipes;
3930         enum pipe pipe = crtc->pipe;
3931
3932         enabled_bigjoiner_pipes(i915, &master_pipes, &slave_pipes);
3933
3934         if (((master_pipes | slave_pipes) & BIT(pipe)) == 0)
3935                 return;
3936
3937         crtc_state->bigjoiner_pipes =
3938                 BIT(get_bigjoiner_master_pipe(pipe, master_pipes, slave_pipes)) |
3939                 get_bigjoiner_slave_pipes(pipe, master_pipes, slave_pipes);
3940 }
3941
3942 static bool hsw_get_pipe_config(struct intel_crtc *crtc,
3943                                 struct intel_crtc_state *pipe_config)
3944 {
3945         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3946         bool active;
3947         u32 tmp;
3948
3949         if (!intel_display_power_get_in_set_if_enabled(dev_priv, &crtc->hw_readout_power_domains,
3950                                                        POWER_DOMAIN_PIPE(crtc->pipe)))
3951                 return false;
3952
3953         pipe_config->shared_dpll = NULL;
3954
3955         active = hsw_get_transcoder_state(crtc, pipe_config, &crtc->hw_readout_power_domains);
3956
3957         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
3958             bxt_get_dsi_transcoder_state(crtc, pipe_config, &crtc->hw_readout_power_domains)) {
3959                 drm_WARN_ON(&dev_priv->drm, active);
3960                 active = true;
3961         }
3962
3963         if (!active)
3964                 goto out;
3965
3966         intel_dsc_get_config(pipe_config);
3967         intel_bigjoiner_get_config(pipe_config);
3968
3969         if (!transcoder_is_dsi(pipe_config->cpu_transcoder) ||
3970             DISPLAY_VER(dev_priv) >= 11)
3971                 intel_get_transcoder_timings(crtc, pipe_config);
3972
3973         if (HAS_VRR(dev_priv) && !transcoder_is_dsi(pipe_config->cpu_transcoder))
3974                 intel_vrr_get_config(crtc, pipe_config);
3975
3976         intel_get_pipe_src_size(crtc, pipe_config);
3977
3978         if (IS_HASWELL(dev_priv)) {
3979                 u32 tmp = intel_de_read(dev_priv,
3980                                         TRANSCONF(pipe_config->cpu_transcoder));
3981
3982                 if (tmp & TRANSCONF_OUTPUT_COLORSPACE_YUV_HSW)
3983                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_YCBCR444;
3984                 else
3985                         pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
3986         } else {
3987                 pipe_config->output_format =
3988                         bdw_get_pipe_misc_output_format(crtc);
3989         }
3990
3991         pipe_config->gamma_mode = intel_de_read(dev_priv,
3992                                                 GAMMA_MODE(crtc->pipe));
3993
3994         pipe_config->csc_mode = intel_de_read(dev_priv,
3995                                               PIPE_CSC_MODE(crtc->pipe));
3996
3997         if (DISPLAY_VER(dev_priv) >= 9) {
3998                 tmp = intel_de_read(dev_priv, SKL_BOTTOM_COLOR(crtc->pipe));
3999
4000                 if (tmp & SKL_BOTTOM_COLOR_GAMMA_ENABLE)
4001                         pipe_config->gamma_enable = true;
4002
4003                 if (tmp & SKL_BOTTOM_COLOR_CSC_ENABLE)
4004                         pipe_config->csc_enable = true;
4005         } else {
4006                 i9xx_get_pipe_color_config(pipe_config);
4007         }
4008
4009         intel_color_get_config(pipe_config);
4010
4011         tmp = intel_de_read(dev_priv, WM_LINETIME(crtc->pipe));
4012         pipe_config->linetime = REG_FIELD_GET(HSW_LINETIME_MASK, tmp);
4013         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
4014                 pipe_config->ips_linetime =
4015                         REG_FIELD_GET(HSW_IPS_LINETIME_MASK, tmp);
4016
4017         if (intel_display_power_get_in_set_if_enabled(dev_priv, &crtc->hw_readout_power_domains,
4018                                                       POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe))) {
4019                 if (DISPLAY_VER(dev_priv) >= 9)
4020                         skl_get_pfit_config(pipe_config);
4021                 else
4022                         ilk_get_pfit_config(pipe_config);
4023         }
4024
4025         hsw_ips_get_config(pipe_config);
4026
4027         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
4028             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
4029                 pipe_config->pixel_multiplier =
4030                         intel_de_read(dev_priv,
4031                                       TRANS_MULT(pipe_config->cpu_transcoder)) + 1;
4032         } else {
4033                 pipe_config->pixel_multiplier = 1;
4034         }
4035
4036         if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) {
4037                 tmp = intel_de_read(dev_priv, DISPLAY_VER(dev_priv) >= 14 ?
4038                                     MTL_CHICKEN_TRANS(pipe_config->cpu_transcoder) :
4039                                     CHICKEN_TRANS(pipe_config->cpu_transcoder));
4040
4041                 pipe_config->framestart_delay = REG_FIELD_GET(HSW_FRAME_START_DELAY_MASK, tmp) + 1;
4042         } else {
4043                 /* no idea if this is correct */
4044                 pipe_config->framestart_delay = 1;
4045         }
4046
4047 out:
4048         intel_display_power_put_all_in_set(dev_priv, &crtc->hw_readout_power_domains);
4049
4050         return active;
4051 }
4052
4053 bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state)
4054 {
4055         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4056         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
4057
4058         if (!i915->display.funcs.display->get_pipe_config(crtc, crtc_state))
4059                 return false;
4060
4061         crtc_state->hw.active = true;
4062
4063         intel_crtc_readout_derived_state(crtc_state);
4064
4065         return true;
4066 }
4067
4068 /* VESA 640x480x72Hz mode to set on the pipe */
4069 static const struct drm_display_mode load_detect_mode = {
4070         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
4071                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
4072 };
4073
4074 static int intel_modeset_disable_planes(struct drm_atomic_state *state,
4075                                         struct drm_crtc *crtc)
4076 {
4077         struct drm_plane *plane;
4078         struct drm_plane_state *plane_state;
4079         int ret, i;
4080
4081         ret = drm_atomic_add_affected_planes(state, crtc);
4082         if (ret)
4083                 return ret;
4084
4085         for_each_new_plane_in_state(state, plane, plane_state, i) {
4086                 if (plane_state->crtc != crtc)
4087                         continue;
4088
4089                 ret = drm_atomic_set_crtc_for_plane(plane_state, NULL);
4090                 if (ret)
4091                         return ret;
4092
4093                 drm_atomic_set_fb_for_plane(plane_state, NULL);
4094         }
4095
4096         return 0;
4097 }
4098
4099 int intel_get_load_detect_pipe(struct drm_connector *connector,
4100                                struct intel_load_detect_pipe *old,
4101                                struct drm_modeset_acquire_ctx *ctx)
4102 {
4103         struct intel_encoder *encoder =
4104                 intel_attached_encoder(to_intel_connector(connector));
4105         struct intel_crtc *possible_crtc;
4106         struct intel_crtc *crtc = NULL;
4107         struct drm_device *dev = encoder->base.dev;
4108         struct drm_i915_private *dev_priv = to_i915(dev);
4109         struct drm_mode_config *config = &dev->mode_config;
4110         struct drm_atomic_state *state = NULL, *restore_state = NULL;
4111         struct drm_connector_state *connector_state;
4112         struct intel_crtc_state *crtc_state;
4113         int ret;
4114
4115         drm_dbg_kms(&dev_priv->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
4116                     connector->base.id, connector->name,
4117                     encoder->base.base.id, encoder->base.name);
4118
4119         old->restore_state = NULL;
4120
4121         drm_WARN_ON(dev, !drm_modeset_is_locked(&config->connection_mutex));
4122
4123         /*
4124          * Algorithm gets a little messy:
4125          *
4126          *   - if the connector already has an assigned crtc, use it (but make
4127          *     sure it's on first)
4128          *
4129          *   - try to find the first unused crtc that can drive this connector,
4130          *     and use that if we find one
4131          */
4132
4133         /* See if we already have a CRTC for this connector */
4134         if (connector->state->crtc) {
4135                 crtc = to_intel_crtc(connector->state->crtc);
4136
4137                 ret = drm_modeset_lock(&crtc->base.mutex, ctx);
4138                 if (ret)
4139                         goto fail;
4140
4141                 /* Make sure the crtc and connector are running */
4142                 goto found;
4143         }
4144
4145         /* Find an unused one (if possible) */
4146         for_each_intel_crtc(dev, possible_crtc) {
4147                 if (!(encoder->base.possible_crtcs &
4148                       drm_crtc_mask(&possible_crtc->base)))
4149                         continue;
4150
4151                 ret = drm_modeset_lock(&possible_crtc->base.mutex, ctx);
4152                 if (ret)
4153                         goto fail;
4154
4155                 if (possible_crtc->base.state->enable) {
4156                         drm_modeset_unlock(&possible_crtc->base.mutex);
4157                         continue;
4158                 }
4159
4160                 crtc = possible_crtc;
4161                 break;
4162         }
4163
4164         /*
4165          * If we didn't find an unused CRTC, don't use any.
4166          */
4167         if (!crtc) {
4168                 drm_dbg_kms(&dev_priv->drm,
4169                             "no pipe available for load-detect\n");
4170                 ret = -ENODEV;
4171                 goto fail;
4172         }
4173
4174 found:
4175         state = drm_atomic_state_alloc(dev);
4176         restore_state = drm_atomic_state_alloc(dev);
4177         if (!state || !restore_state) {
4178                 ret = -ENOMEM;
4179                 goto fail;
4180         }
4181
4182         state->acquire_ctx = ctx;
4183         restore_state->acquire_ctx = ctx;
4184
4185         connector_state = drm_atomic_get_connector_state(state, connector);
4186         if (IS_ERR(connector_state)) {
4187                 ret = PTR_ERR(connector_state);
4188                 goto fail;
4189         }
4190
4191         ret = drm_atomic_set_crtc_for_connector(connector_state, &crtc->base);
4192         if (ret)
4193                 goto fail;
4194
4195         crtc_state = intel_atomic_get_crtc_state(state, crtc);
4196         if (IS_ERR(crtc_state)) {
4197                 ret = PTR_ERR(crtc_state);
4198                 goto fail;
4199         }
4200
4201         crtc_state->uapi.active = true;
4202
4203         ret = drm_atomic_set_mode_for_crtc(&crtc_state->uapi,
4204                                            &load_detect_mode);
4205         if (ret)
4206                 goto fail;
4207
4208         ret = intel_modeset_disable_planes(state, &crtc->base);
4209         if (ret)
4210                 goto fail;
4211
4212         ret = PTR_ERR_OR_ZERO(drm_atomic_get_connector_state(restore_state, connector));
4213         if (!ret)
4214                 ret = PTR_ERR_OR_ZERO(drm_atomic_get_crtc_state(restore_state, &crtc->base));
4215         if (!ret)
4216                 ret = drm_atomic_add_affected_planes(restore_state, &crtc->base);
4217         if (ret) {
4218                 drm_dbg_kms(&dev_priv->drm,
4219                             "Failed to create a copy of old state to restore: %i\n",
4220                             ret);
4221                 goto fail;
4222         }
4223
4224         ret = drm_atomic_commit(state);
4225         if (ret) {
4226                 drm_dbg_kms(&dev_priv->drm,
4227                             "failed to set mode on load-detect pipe\n");
4228                 goto fail;
4229         }
4230
4231         old->restore_state = restore_state;
4232         drm_atomic_state_put(state);
4233
4234         /* let the connector get through one full cycle before testing */
4235         intel_crtc_wait_for_next_vblank(crtc);
4236
4237         return true;
4238
4239 fail:
4240         if (state) {
4241                 drm_atomic_state_put(state);
4242                 state = NULL;
4243         }
4244         if (restore_state) {
4245                 drm_atomic_state_put(restore_state);
4246                 restore_state = NULL;
4247         }
4248
4249         if (ret == -EDEADLK)
4250                 return ret;
4251
4252         return false;
4253 }
4254
4255 void intel_release_load_detect_pipe(struct drm_connector *connector,
4256                                     struct intel_load_detect_pipe *old,
4257                                     struct drm_modeset_acquire_ctx *ctx)
4258 {
4259         struct intel_encoder *intel_encoder =
4260                 intel_attached_encoder(to_intel_connector(connector));
4261         struct drm_i915_private *i915 = to_i915(intel_encoder->base.dev);
4262         struct drm_encoder *encoder = &intel_encoder->base;
4263         struct drm_atomic_state *state = old->restore_state;
4264         int ret;
4265
4266         drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
4267                     connector->base.id, connector->name,
4268                     encoder->base.id, encoder->name);
4269
4270         if (!state)
4271                 return;
4272
4273         ret = drm_atomic_helper_commit_duplicated_state(state, ctx);
4274         if (ret)
4275                 drm_dbg_kms(&i915->drm,
4276                             "Couldn't release load detect pipe: %i\n", ret);
4277         drm_atomic_state_put(state);
4278 }
4279
4280 static int i9xx_pll_refclk(struct drm_device *dev,
4281                            const struct intel_crtc_state *pipe_config)
4282 {
4283         struct drm_i915_private *dev_priv = to_i915(dev);
4284         u32 dpll = pipe_config->dpll_hw_state.dpll;
4285
4286         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
4287                 return dev_priv->display.vbt.lvds_ssc_freq;
4288         else if (HAS_PCH_SPLIT(dev_priv))
4289                 return 120000;
4290         else if (DISPLAY_VER(dev_priv) != 2)
4291                 return 96000;
4292         else
4293                 return 48000;
4294 }
4295
4296 /* Returns the clock of the currently programmed mode of the given pipe. */
4297 void i9xx_crtc_clock_get(struct intel_crtc *crtc,
4298                          struct intel_crtc_state *pipe_config)
4299 {
4300         struct drm_device *dev = crtc->base.dev;
4301         struct drm_i915_private *dev_priv = to_i915(dev);
4302         u32 dpll = pipe_config->dpll_hw_state.dpll;
4303         u32 fp;
4304         struct dpll clock;
4305         int port_clock;
4306         int refclk = i9xx_pll_refclk(dev, pipe_config);
4307
4308         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
4309                 fp = pipe_config->dpll_hw_state.fp0;
4310         else
4311                 fp = pipe_config->dpll_hw_state.fp1;
4312
4313         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
4314         if (IS_PINEVIEW(dev_priv)) {
4315                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
4316                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
4317         } else {
4318                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
4319                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
4320         }
4321
4322         if (DISPLAY_VER(dev_priv) != 2) {
4323                 if (IS_PINEVIEW(dev_priv))
4324                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
4325                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
4326                 else
4327                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
4328                                DPLL_FPA01_P1_POST_DIV_SHIFT);
4329
4330                 switch (dpll & DPLL_MODE_MASK) {
4331                 case DPLLB_MODE_DAC_SERIAL:
4332                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
4333                                 5 : 10;
4334                         break;
4335                 case DPLLB_MODE_LVDS:
4336                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
4337                                 7 : 14;
4338                         break;
4339                 default:
4340                         drm_dbg_kms(&dev_priv->drm,
4341                                     "Unknown DPLL mode %08x in programmed "
4342                                     "mode\n", (int)(dpll & DPLL_MODE_MASK));
4343                         return;
4344                 }
4345
4346                 if (IS_PINEVIEW(dev_priv))
4347                         port_clock = pnv_calc_dpll_params(refclk, &clock);
4348                 else
4349                         port_clock = i9xx_calc_dpll_params(refclk, &clock);
4350         } else {
4351                 enum pipe lvds_pipe;
4352
4353                 if (IS_I85X(dev_priv) &&
4354                     intel_lvds_port_enabled(dev_priv, LVDS, &lvds_pipe) &&
4355                     lvds_pipe == crtc->pipe) {
4356                         u32 lvds = intel_de_read(dev_priv, LVDS);
4357
4358                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
4359                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
4360
4361                         if (lvds & LVDS_CLKB_POWER_UP)
4362                                 clock.p2 = 7;
4363                         else
4364                                 clock.p2 = 14;
4365                 } else {
4366                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
4367                                 clock.p1 = 2;
4368                         else {
4369                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
4370                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
4371                         }
4372                         if (dpll & PLL_P2_DIVIDE_BY_4)
4373                                 clock.p2 = 4;
4374                         else
4375                                 clock.p2 = 2;
4376                 }
4377
4378                 port_clock = i9xx_calc_dpll_params(refclk, &clock);
4379         }
4380
4381         /*
4382          * This value includes pixel_multiplier. We will use
4383          * port_clock to compute adjusted_mode.crtc_clock in the
4384          * encoder's get_config() function.
4385          */
4386         pipe_config->port_clock = port_clock;
4387 }
4388
4389 int intel_dotclock_calculate(int link_freq,
4390                              const struct intel_link_m_n *m_n)
4391 {
4392         /*
4393          * The calculation for the data clock is:
4394          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
4395          * But we want to avoid losing precison if possible, so:
4396          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
4397          *
4398          * and the link clock is simpler:
4399          * link_clock = (m * link_clock) / n
4400          */
4401
4402         if (!m_n->link_n)
4403                 return 0;
4404
4405         return DIV_ROUND_UP_ULL(mul_u32_u32(m_n->link_m, link_freq),
4406                                 m_n->link_n);
4407 }
4408
4409 int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config)
4410 {
4411         int dotclock;
4412
4413         if (intel_crtc_has_dp_encoder(pipe_config))
4414                 dotclock = intel_dotclock_calculate(pipe_config->port_clock,
4415                                                     &pipe_config->dp_m_n);
4416         else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp > 24)
4417                 dotclock = DIV_ROUND_CLOSEST(pipe_config->port_clock * 24,
4418                                              pipe_config->pipe_bpp);
4419         else
4420                 dotclock = pipe_config->port_clock;
4421
4422         if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 &&
4423             !intel_crtc_has_dp_encoder(pipe_config))
4424                 dotclock *= 2;
4425
4426         if (pipe_config->pixel_multiplier)
4427                 dotclock /= pipe_config->pixel_multiplier;
4428
4429         return dotclock;
4430 }
4431
4432 /* Returns the currently programmed mode of the given encoder. */
4433 struct drm_display_mode *
4434 intel_encoder_current_mode(struct intel_encoder *encoder)
4435 {
4436         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
4437         struct intel_crtc_state *crtc_state;
4438         struct drm_display_mode *mode;
4439         struct intel_crtc *crtc;
4440         enum pipe pipe;
4441
4442         if (!encoder->get_hw_state(encoder, &pipe))
4443                 return NULL;
4444
4445         crtc = intel_crtc_for_pipe(dev_priv, pipe);
4446
4447         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
4448         if (!mode)
4449                 return NULL;
4450
4451         crtc_state = intel_crtc_state_alloc(crtc);
4452         if (!crtc_state) {
4453                 kfree(mode);
4454                 return NULL;
4455         }
4456
4457         if (!intel_crtc_get_pipe_config(crtc_state)) {
4458                 kfree(crtc_state);
4459                 kfree(mode);
4460                 return NULL;
4461         }
4462
4463         intel_encoder_get_config(encoder, crtc_state);
4464
4465         intel_mode_from_crtc_timings(mode, &crtc_state->hw.adjusted_mode);
4466
4467         kfree(crtc_state);
4468
4469         return mode;
4470 }
4471
4472 static bool encoders_cloneable(const struct intel_encoder *a,
4473                                const struct intel_encoder *b)
4474 {
4475         /* masks could be asymmetric, so check both ways */
4476         return a == b || (a->cloneable & BIT(b->type) &&
4477                           b->cloneable & BIT(a->type));
4478 }
4479
4480 static bool check_single_encoder_cloning(struct intel_atomic_state *state,
4481                                          struct intel_crtc *crtc,
4482                                          struct intel_encoder *encoder)
4483 {
4484         struct intel_encoder *source_encoder;
4485         struct drm_connector *connector;
4486         struct drm_connector_state *connector_state;
4487         int i;
4488
4489         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
4490                 if (connector_state->crtc != &crtc->base)
4491                         continue;
4492
4493                 source_encoder =
4494                         to_intel_encoder(connector_state->best_encoder);
4495                 if (!encoders_cloneable(encoder, source_encoder))
4496                         return false;
4497         }
4498
4499         return true;
4500 }
4501
4502 static int icl_add_linked_planes(struct intel_atomic_state *state)
4503 {
4504         struct intel_plane *plane, *linked;
4505         struct intel_plane_state *plane_state, *linked_plane_state;
4506         int i;
4507
4508         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4509                 linked = plane_state->planar_linked_plane;
4510
4511                 if (!linked)
4512                         continue;
4513
4514                 linked_plane_state = intel_atomic_get_plane_state(state, linked);
4515                 if (IS_ERR(linked_plane_state))
4516                         return PTR_ERR(linked_plane_state);
4517
4518                 drm_WARN_ON(state->base.dev,
4519                             linked_plane_state->planar_linked_plane != plane);
4520                 drm_WARN_ON(state->base.dev,
4521                             linked_plane_state->planar_slave == plane_state->planar_slave);
4522         }
4523
4524         return 0;
4525 }
4526
4527 static int icl_check_nv12_planes(struct intel_crtc_state *crtc_state)
4528 {
4529         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4530         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4531         struct intel_atomic_state *state = to_intel_atomic_state(crtc_state->uapi.state);
4532         struct intel_plane *plane, *linked;
4533         struct intel_plane_state *plane_state;
4534         int i;
4535
4536         if (DISPLAY_VER(dev_priv) < 11)
4537                 return 0;
4538
4539         /*
4540          * Destroy all old plane links and make the slave plane invisible
4541          * in the crtc_state->active_planes mask.
4542          */
4543         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4544                 if (plane->pipe != crtc->pipe || !plane_state->planar_linked_plane)
4545                         continue;
4546
4547                 plane_state->planar_linked_plane = NULL;
4548                 if (plane_state->planar_slave && !plane_state->uapi.visible) {
4549                         crtc_state->enabled_planes &= ~BIT(plane->id);
4550                         crtc_state->active_planes &= ~BIT(plane->id);
4551                         crtc_state->update_planes |= BIT(plane->id);
4552                         crtc_state->data_rate[plane->id] = 0;
4553                         crtc_state->rel_data_rate[plane->id] = 0;
4554                 }
4555
4556                 plane_state->planar_slave = false;
4557         }
4558
4559         if (!crtc_state->nv12_planes)
4560                 return 0;
4561
4562         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4563                 struct intel_plane_state *linked_state = NULL;
4564
4565                 if (plane->pipe != crtc->pipe ||
4566                     !(crtc_state->nv12_planes & BIT(plane->id)))
4567                         continue;
4568
4569                 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, linked) {
4570                         if (!icl_is_nv12_y_plane(dev_priv, linked->id))
4571                                 continue;
4572
4573                         if (crtc_state->active_planes & BIT(linked->id))
4574                                 continue;
4575
4576                         linked_state = intel_atomic_get_plane_state(state, linked);
4577                         if (IS_ERR(linked_state))
4578                                 return PTR_ERR(linked_state);
4579
4580                         break;
4581                 }
4582
4583                 if (!linked_state) {
4584                         drm_dbg_kms(&dev_priv->drm,
4585                                     "Need %d free Y planes for planar YUV\n",
4586                                     hweight8(crtc_state->nv12_planes));
4587
4588                         return -EINVAL;
4589                 }
4590
4591                 plane_state->planar_linked_plane = linked;
4592
4593                 linked_state->planar_slave = true;
4594                 linked_state->planar_linked_plane = plane;
4595                 crtc_state->enabled_planes |= BIT(linked->id);
4596                 crtc_state->active_planes |= BIT(linked->id);
4597                 crtc_state->update_planes |= BIT(linked->id);
4598                 crtc_state->data_rate[linked->id] =
4599                         crtc_state->data_rate_y[plane->id];
4600                 crtc_state->rel_data_rate[linked->id] =
4601                         crtc_state->rel_data_rate_y[plane->id];
4602                 drm_dbg_kms(&dev_priv->drm, "Using %s as Y plane for %s\n",
4603                             linked->base.name, plane->base.name);
4604
4605                 /* Copy parameters to slave plane */
4606                 linked_state->ctl = plane_state->ctl | PLANE_CTL_YUV420_Y_PLANE;
4607                 linked_state->color_ctl = plane_state->color_ctl;
4608                 linked_state->view = plane_state->view;
4609                 linked_state->decrypt = plane_state->decrypt;
4610
4611                 intel_plane_copy_hw_state(linked_state, plane_state);
4612                 linked_state->uapi.src = plane_state->uapi.src;
4613                 linked_state->uapi.dst = plane_state->uapi.dst;
4614
4615                 if (icl_is_hdr_plane(dev_priv, plane->id)) {
4616                         if (linked->id == PLANE_SPRITE5)
4617                                 plane_state->cus_ctl |= PLANE_CUS_Y_PLANE_7_ICL;
4618                         else if (linked->id == PLANE_SPRITE4)
4619                                 plane_state->cus_ctl |= PLANE_CUS_Y_PLANE_6_ICL;
4620                         else if (linked->id == PLANE_SPRITE3)
4621                                 plane_state->cus_ctl |= PLANE_CUS_Y_PLANE_5_RKL;
4622                         else if (linked->id == PLANE_SPRITE2)
4623                                 plane_state->cus_ctl |= PLANE_CUS_Y_PLANE_4_RKL;
4624                         else
4625                                 MISSING_CASE(linked->id);
4626                 }
4627         }
4628
4629         return 0;
4630 }
4631
4632 static bool c8_planes_changed(const struct intel_crtc_state *new_crtc_state)
4633 {
4634         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
4635         struct intel_atomic_state *state =
4636                 to_intel_atomic_state(new_crtc_state->uapi.state);
4637         const struct intel_crtc_state *old_crtc_state =
4638                 intel_atomic_get_old_crtc_state(state, crtc);
4639
4640         return !old_crtc_state->c8_planes != !new_crtc_state->c8_planes;
4641 }
4642
4643 static u16 hsw_linetime_wm(const struct intel_crtc_state *crtc_state)
4644 {
4645         const struct drm_display_mode *pipe_mode =
4646                 &crtc_state->hw.pipe_mode;
4647         int linetime_wm;
4648
4649         if (!crtc_state->hw.enable)
4650                 return 0;
4651
4652         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
4653                                         pipe_mode->crtc_clock);
4654
4655         return min(linetime_wm, 0x1ff);
4656 }
4657
4658 static u16 hsw_ips_linetime_wm(const struct intel_crtc_state *crtc_state,
4659                                const struct intel_cdclk_state *cdclk_state)
4660 {
4661         const struct drm_display_mode *pipe_mode =
4662                 &crtc_state->hw.pipe_mode;
4663         int linetime_wm;
4664
4665         if (!crtc_state->hw.enable)
4666                 return 0;
4667
4668         linetime_wm = DIV_ROUND_CLOSEST(pipe_mode->crtc_htotal * 1000 * 8,
4669                                         cdclk_state->logical.cdclk);
4670
4671         return min(linetime_wm, 0x1ff);
4672 }
4673
4674 static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state)
4675 {
4676         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4677         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4678         const struct drm_display_mode *pipe_mode =
4679                 &crtc_state->hw.pipe_mode;
4680         int linetime_wm;
4681
4682         if (!crtc_state->hw.enable)
4683                 return 0;
4684
4685         linetime_wm = DIV_ROUND_UP(pipe_mode->crtc_htotal * 1000 * 8,
4686                                    crtc_state->pixel_rate);
4687
4688         /* Display WA #1135: BXT:ALL GLK:ALL */
4689         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
4690             skl_watermark_ipc_enabled(dev_priv))
4691                 linetime_wm /= 2;
4692
4693         return min(linetime_wm, 0x1ff);
4694 }
4695
4696 static int hsw_compute_linetime_wm(struct intel_atomic_state *state,
4697                                    struct intel_crtc *crtc)
4698 {
4699         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4700         struct intel_crtc_state *crtc_state =
4701                 intel_atomic_get_new_crtc_state(state, crtc);
4702         const struct intel_cdclk_state *cdclk_state;
4703
4704         if (DISPLAY_VER(dev_priv) >= 9)
4705                 crtc_state->linetime = skl_linetime_wm(crtc_state);
4706         else
4707                 crtc_state->linetime = hsw_linetime_wm(crtc_state);
4708
4709         if (!hsw_crtc_supports_ips(crtc))
4710                 return 0;
4711
4712         cdclk_state = intel_atomic_get_cdclk_state(state);
4713         if (IS_ERR(cdclk_state))
4714                 return PTR_ERR(cdclk_state);
4715
4716         crtc_state->ips_linetime = hsw_ips_linetime_wm(crtc_state,
4717                                                        cdclk_state);
4718
4719         return 0;
4720 }
4721
4722 static int intel_crtc_atomic_check(struct intel_atomic_state *state,
4723                                    struct intel_crtc *crtc)
4724 {
4725         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4726         struct intel_crtc_state *crtc_state =
4727                 intel_atomic_get_new_crtc_state(state, crtc);
4728         int ret;
4729
4730         if (DISPLAY_VER(dev_priv) < 5 && !IS_G4X(dev_priv) &&
4731             intel_crtc_needs_modeset(crtc_state) &&
4732             !crtc_state->hw.active)
4733                 crtc_state->update_wm_post = true;
4734
4735         if (intel_crtc_needs_modeset(crtc_state)) {
4736                 ret = intel_dpll_crtc_get_shared_dpll(state, crtc);
4737                 if (ret)
4738                         return ret;
4739         }
4740
4741         /*
4742          * May need to update pipe gamma enable bits
4743          * when C8 planes are getting enabled/disabled.
4744          */
4745         if (c8_planes_changed(crtc_state))
4746                 crtc_state->uapi.color_mgmt_changed = true;
4747
4748         if (intel_crtc_needs_color_update(crtc_state)) {
4749                 ret = intel_color_check(crtc_state);
4750                 if (ret)
4751                         return ret;
4752         }
4753
4754         ret = intel_compute_pipe_wm(state, crtc);
4755         if (ret) {
4756                 drm_dbg_kms(&dev_priv->drm,
4757                             "Target pipe watermarks are invalid\n");
4758                 return ret;
4759         }
4760
4761         /*
4762          * Calculate 'intermediate' watermarks that satisfy both the
4763          * old state and the new state.  We can program these
4764          * immediately.
4765          */
4766         ret = intel_compute_intermediate_wm(state, crtc);
4767         if (ret) {
4768                 drm_dbg_kms(&dev_priv->drm,
4769                             "No valid intermediate pipe watermarks are possible\n");
4770                 return ret;
4771         }
4772
4773         if (DISPLAY_VER(dev_priv) >= 9) {
4774                 if (intel_crtc_needs_modeset(crtc_state) ||
4775                     intel_crtc_needs_fastset(crtc_state)) {
4776                         ret = skl_update_scaler_crtc(crtc_state);
4777                         if (ret)
4778                                 return ret;
4779                 }
4780
4781                 ret = intel_atomic_setup_scalers(dev_priv, crtc, crtc_state);
4782                 if (ret)
4783                         return ret;
4784         }
4785
4786         if (HAS_IPS(dev_priv)) {
4787                 ret = hsw_ips_compute_config(state, crtc);
4788                 if (ret)
4789                         return ret;
4790         }
4791
4792         if (DISPLAY_VER(dev_priv) >= 9 ||
4793             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
4794                 ret = hsw_compute_linetime_wm(state, crtc);
4795                 if (ret)
4796                         return ret;
4797
4798         }
4799
4800         ret = intel_psr2_sel_fetch_update(state, crtc);
4801         if (ret)
4802                 return ret;
4803
4804         return 0;
4805 }
4806
4807 static int
4808 compute_sink_pipe_bpp(const struct drm_connector_state *conn_state,
4809                       struct intel_crtc_state *crtc_state)
4810 {
4811         struct drm_connector *connector = conn_state->connector;
4812         struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
4813         const struct drm_display_info *info = &connector->display_info;
4814         int bpp;
4815
4816         switch (conn_state->max_bpc) {
4817         case 6 ... 7:
4818                 bpp = 6 * 3;
4819                 break;
4820         case 8 ... 9:
4821                 bpp = 8 * 3;
4822                 break;
4823         case 10 ... 11:
4824                 bpp = 10 * 3;
4825                 break;
4826         case 12 ... 16:
4827                 bpp = 12 * 3;
4828                 break;
4829         default:
4830                 MISSING_CASE(conn_state->max_bpc);
4831                 return -EINVAL;
4832         }
4833
4834         if (bpp < crtc_state->pipe_bpp) {
4835                 drm_dbg_kms(&i915->drm,
4836                             "[CONNECTOR:%d:%s] Limiting display bpp to %d "
4837                             "(EDID bpp %d, max requested bpp %d, max platform bpp %d)\n",
4838                             connector->base.id, connector->name,
4839                             bpp, 3 * info->bpc,
4840                             3 * conn_state->max_requested_bpc,
4841                             crtc_state->pipe_bpp);
4842
4843                 crtc_state->pipe_bpp = bpp;
4844         }
4845
4846         return 0;
4847 }
4848
4849 static int
4850 compute_baseline_pipe_bpp(struct intel_atomic_state *state,
4851                           struct intel_crtc *crtc)
4852 {
4853         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4854         struct intel_crtc_state *crtc_state =
4855                 intel_atomic_get_new_crtc_state(state, crtc);
4856         struct drm_connector *connector;
4857         struct drm_connector_state *connector_state;
4858         int bpp, i;
4859
4860         if ((IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
4861             IS_CHERRYVIEW(dev_priv)))
4862                 bpp = 10*3;
4863         else if (DISPLAY_VER(dev_priv) >= 5)
4864                 bpp = 12*3;
4865         else
4866                 bpp = 8*3;
4867
4868         crtc_state->pipe_bpp = bpp;
4869
4870         /* Clamp display bpp to connector max bpp */
4871         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
4872                 int ret;
4873
4874                 if (connector_state->crtc != &crtc->base)
4875                         continue;
4876
4877                 ret = compute_sink_pipe_bpp(connector_state, crtc_state);
4878                 if (ret)
4879                         return ret;
4880         }
4881
4882         return 0;
4883 }
4884
4885 static bool check_digital_port_conflicts(struct intel_atomic_state *state)
4886 {
4887         struct drm_device *dev = state->base.dev;
4888         struct drm_connector *connector;
4889         struct drm_connector_list_iter conn_iter;
4890         unsigned int used_ports = 0;
4891         unsigned int used_mst_ports = 0;
4892         bool ret = true;
4893
4894         /*
4895          * We're going to peek into connector->state,
4896          * hence connection_mutex must be held.
4897          */
4898         drm_modeset_lock_assert_held(&dev->mode_config.connection_mutex);
4899
4900         /*
4901          * Walk the connector list instead of the encoder
4902          * list to detect the problem on ddi platforms
4903          * where there's just one encoder per digital port.
4904          */
4905         drm_connector_list_iter_begin(dev, &conn_iter);
4906         drm_for_each_connector_iter(connector, &conn_iter) {
4907                 struct drm_connector_state *connector_state;
4908                 struct intel_encoder *encoder;
4909
4910                 connector_state =
4911                         drm_atomic_get_new_connector_state(&state->base,
4912                                                            connector);
4913                 if (!connector_state)
4914                         connector_state = connector->state;
4915
4916                 if (!connector_state->best_encoder)
4917                         continue;
4918
4919                 encoder = to_intel_encoder(connector_state->best_encoder);
4920
4921                 drm_WARN_ON(dev, !connector_state->crtc);
4922
4923                 switch (encoder->type) {
4924                 case INTEL_OUTPUT_DDI:
4925                         if (drm_WARN_ON(dev, !HAS_DDI(to_i915(dev))))
4926                                 break;
4927                         fallthrough;
4928                 case INTEL_OUTPUT_DP:
4929                 case INTEL_OUTPUT_HDMI:
4930                 case INTEL_OUTPUT_EDP:
4931                         /* the same port mustn't appear more than once */
4932                         if (used_ports & BIT(encoder->port))
4933                                 ret = false;
4934
4935                         used_ports |= BIT(encoder->port);
4936                         break;
4937                 case INTEL_OUTPUT_DP_MST:
4938                         used_mst_ports |=
4939                                 1 << encoder->port;
4940                         break;
4941                 default:
4942                         break;
4943                 }
4944         }
4945         drm_connector_list_iter_end(&conn_iter);
4946
4947         /* can't mix MST and SST/HDMI on the same port */
4948         if (used_ports & used_mst_ports)
4949                 return false;
4950
4951         return ret;
4952 }
4953
4954 static void
4955 intel_crtc_copy_uapi_to_hw_state_nomodeset(struct intel_atomic_state *state,
4956                                            struct intel_crtc *crtc)
4957 {
4958         struct intel_crtc_state *crtc_state =
4959                 intel_atomic_get_new_crtc_state(state, crtc);
4960
4961         WARN_ON(intel_crtc_is_bigjoiner_slave(crtc_state));
4962
4963         drm_property_replace_blob(&crtc_state->hw.degamma_lut,
4964                                   crtc_state->uapi.degamma_lut);
4965         drm_property_replace_blob(&crtc_state->hw.gamma_lut,
4966                                   crtc_state->uapi.gamma_lut);
4967         drm_property_replace_blob(&crtc_state->hw.ctm,
4968                                   crtc_state->uapi.ctm);
4969 }
4970
4971 static void
4972 intel_crtc_copy_uapi_to_hw_state_modeset(struct intel_atomic_state *state,
4973                                          struct intel_crtc *crtc)
4974 {
4975         struct intel_crtc_state *crtc_state =
4976                 intel_atomic_get_new_crtc_state(state, crtc);
4977
4978         WARN_ON(intel_crtc_is_bigjoiner_slave(crtc_state));
4979
4980         crtc_state->hw.enable = crtc_state->uapi.enable;
4981         crtc_state->hw.active = crtc_state->uapi.active;
4982         drm_mode_copy(&crtc_state->hw.mode,
4983                       &crtc_state->uapi.mode);
4984         drm_mode_copy(&crtc_state->hw.adjusted_mode,
4985                       &crtc_state->uapi.adjusted_mode);
4986         crtc_state->hw.scaling_filter = crtc_state->uapi.scaling_filter;
4987
4988         intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc);
4989 }
4990
4991 static void
4992 copy_bigjoiner_crtc_state_nomodeset(struct intel_atomic_state *state,
4993                                     struct intel_crtc *slave_crtc)
4994 {
4995         struct intel_crtc_state *slave_crtc_state =
4996                 intel_atomic_get_new_crtc_state(state, slave_crtc);
4997         struct intel_crtc *master_crtc = intel_master_crtc(slave_crtc_state);
4998         const struct intel_crtc_state *master_crtc_state =
4999                 intel_atomic_get_new_crtc_state(state, master_crtc);
5000
5001         drm_property_replace_blob(&slave_crtc_state->hw.degamma_lut,
5002                                   master_crtc_state->hw.degamma_lut);
5003         drm_property_replace_blob(&slave_crtc_state->hw.gamma_lut,
5004                                   master_crtc_state->hw.gamma_lut);
5005         drm_property_replace_blob(&slave_crtc_state->hw.ctm,
5006                                   master_crtc_state->hw.ctm);
5007
5008         slave_crtc_state->uapi.color_mgmt_changed = master_crtc_state->uapi.color_mgmt_changed;
5009 }
5010
5011 static int
5012 copy_bigjoiner_crtc_state_modeset(struct intel_atomic_state *state,
5013                                   struct intel_crtc *slave_crtc)
5014 {
5015         struct intel_crtc_state *slave_crtc_state =
5016                 intel_atomic_get_new_crtc_state(state, slave_crtc);
5017         struct intel_crtc *master_crtc = intel_master_crtc(slave_crtc_state);
5018         const struct intel_crtc_state *master_crtc_state =
5019                 intel_atomic_get_new_crtc_state(state, master_crtc);
5020         struct intel_crtc_state *saved_state;
5021
5022         WARN_ON(master_crtc_state->bigjoiner_pipes !=
5023                 slave_crtc_state->bigjoiner_pipes);
5024
5025         saved_state = kmemdup(master_crtc_state, sizeof(*saved_state), GFP_KERNEL);
5026         if (!saved_state)
5027                 return -ENOMEM;
5028
5029         /* preserve some things from the slave's original crtc state */
5030         saved_state->uapi = slave_crtc_state->uapi;
5031         saved_state->scaler_state = slave_crtc_state->scaler_state;
5032         saved_state->shared_dpll = slave_crtc_state->shared_dpll;
5033         saved_state->dpll_hw_state = slave_crtc_state->dpll_hw_state;
5034         saved_state->crc_enabled = slave_crtc_state->crc_enabled;
5035
5036         intel_crtc_free_hw_state(slave_crtc_state);
5037         memcpy(slave_crtc_state, saved_state, sizeof(*slave_crtc_state));
5038         kfree(saved_state);
5039
5040         /* Re-init hw state */
5041         memset(&slave_crtc_state->hw, 0, sizeof(slave_crtc_state->hw));
5042         slave_crtc_state->hw.enable = master_crtc_state->hw.enable;
5043         slave_crtc_state->hw.active = master_crtc_state->hw.active;
5044         drm_mode_copy(&slave_crtc_state->hw.mode,
5045                       &master_crtc_state->hw.mode);
5046         drm_mode_copy(&slave_crtc_state->hw.pipe_mode,
5047                       &master_crtc_state->hw.pipe_mode);
5048         drm_mode_copy(&slave_crtc_state->hw.adjusted_mode,
5049                       &master_crtc_state->hw.adjusted_mode);
5050         slave_crtc_state->hw.scaling_filter = master_crtc_state->hw.scaling_filter;
5051
5052         copy_bigjoiner_crtc_state_nomodeset(state, slave_crtc);
5053
5054         slave_crtc_state->uapi.mode_changed = master_crtc_state->uapi.mode_changed;
5055         slave_crtc_state->uapi.connectors_changed = master_crtc_state->uapi.connectors_changed;
5056         slave_crtc_state->uapi.active_changed = master_crtc_state->uapi.active_changed;
5057
5058         WARN_ON(master_crtc_state->bigjoiner_pipes !=
5059                 slave_crtc_state->bigjoiner_pipes);
5060
5061         return 0;
5062 }
5063
5064 static int
5065 intel_crtc_prepare_cleared_state(struct intel_atomic_state *state,
5066                                  struct intel_crtc *crtc)
5067 {
5068         struct intel_crtc_state *crtc_state =
5069                 intel_atomic_get_new_crtc_state(state, crtc);
5070         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5071         struct intel_crtc_state *saved_state;
5072
5073         saved_state = intel_crtc_state_alloc(crtc);
5074         if (!saved_state)
5075                 return -ENOMEM;
5076
5077         /* free the old crtc_state->hw members */
5078         intel_crtc_free_hw_state(crtc_state);
5079
5080         /* FIXME: before the switch to atomic started, a new pipe_config was
5081          * kzalloc'd. Code that depends on any field being zero should be
5082          * fixed, so that the crtc_state can be safely duplicated. For now,
5083          * only fields that are know to not cause problems are preserved. */
5084
5085         saved_state->uapi = crtc_state->uapi;
5086         saved_state->inherited = crtc_state->inherited;
5087         saved_state->scaler_state = crtc_state->scaler_state;
5088         saved_state->shared_dpll = crtc_state->shared_dpll;
5089         saved_state->dpll_hw_state = crtc_state->dpll_hw_state;
5090         memcpy(saved_state->icl_port_dplls, crtc_state->icl_port_dplls,
5091                sizeof(saved_state->icl_port_dplls));
5092         saved_state->crc_enabled = crtc_state->crc_enabled;
5093         if (IS_G4X(dev_priv) ||
5094             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
5095                 saved_state->wm = crtc_state->wm;
5096
5097         memcpy(crtc_state, saved_state, sizeof(*crtc_state));
5098         kfree(saved_state);
5099
5100         intel_crtc_copy_uapi_to_hw_state_modeset(state, crtc);
5101
5102         return 0;
5103 }
5104
5105 static int
5106 intel_modeset_pipe_config(struct intel_atomic_state *state,
5107                           struct intel_crtc *crtc)
5108 {
5109         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
5110         struct intel_crtc_state *crtc_state =
5111                 intel_atomic_get_new_crtc_state(state, crtc);
5112         struct drm_connector *connector;
5113         struct drm_connector_state *connector_state;
5114         int pipe_src_w, pipe_src_h;
5115         int base_bpp, ret, i;
5116         bool retry = true;
5117
5118         crtc_state->cpu_transcoder = (enum transcoder) crtc->pipe;
5119
5120         crtc_state->framestart_delay = 1;
5121
5122         /*
5123          * Sanitize sync polarity flags based on requested ones. If neither
5124          * positive or negative polarity is requested, treat this as meaning
5125          * negative polarity.
5126          */
5127         if (!(crtc_state->hw.adjusted_mode.flags &
5128               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
5129                 crtc_state->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
5130
5131         if (!(crtc_state->hw.adjusted_mode.flags &
5132               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
5133                 crtc_state->hw.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
5134
5135         ret = compute_baseline_pipe_bpp(state, crtc);
5136         if (ret)
5137                 return ret;
5138
5139         base_bpp = crtc_state->pipe_bpp;
5140
5141         /*
5142          * Determine the real pipe dimensions. Note that stereo modes can
5143          * increase the actual pipe size due to the frame doubling and
5144          * insertion of additional space for blanks between the frame. This
5145          * is stored in the crtc timings. We use the requested mode to do this
5146          * computation to clearly distinguish it from the adjusted mode, which
5147          * can be changed by the connectors in the below retry loop.
5148          */
5149         drm_mode_get_hv_timing(&crtc_state->hw.mode,
5150                                &pipe_src_w, &pipe_src_h);
5151         drm_rect_init(&crtc_state->pipe_src, 0, 0,
5152                       pipe_src_w, pipe_src_h);
5153
5154         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5155                 struct intel_encoder *encoder =
5156                         to_intel_encoder(connector_state->best_encoder);
5157
5158                 if (connector_state->crtc != &crtc->base)
5159                         continue;
5160
5161                 if (!check_single_encoder_cloning(state, crtc, encoder)) {
5162                         drm_dbg_kms(&i915->drm,
5163                                     "[ENCODER:%d:%s] rejecting invalid cloning configuration\n",
5164                                     encoder->base.base.id, encoder->base.name);
5165                         return -EINVAL;
5166                 }
5167
5168                 /*
5169                  * Determine output_types before calling the .compute_config()
5170                  * hooks so that the hooks can use this information safely.
5171                  */
5172                 if (encoder->compute_output_type)
5173                         crtc_state->output_types |=
5174                                 BIT(encoder->compute_output_type(encoder, crtc_state,
5175                                                                  connector_state));
5176                 else
5177                         crtc_state->output_types |= BIT(encoder->type);
5178         }
5179
5180 encoder_retry:
5181         /* Ensure the port clock defaults are reset when retrying. */
5182         crtc_state->port_clock = 0;
5183         crtc_state->pixel_multiplier = 1;
5184
5185         /* Fill in default crtc timings, allow encoders to overwrite them. */
5186         drm_mode_set_crtcinfo(&crtc_state->hw.adjusted_mode,
5187                               CRTC_STEREO_DOUBLE);
5188
5189         /* Pass our mode to the connectors and the CRTC to give them a chance to
5190          * adjust it according to limitations or connector properties, and also
5191          * a chance to reject the mode entirely.
5192          */
5193         for_each_new_connector_in_state(&state->base, connector, connector_state, i) {
5194                 struct intel_encoder *encoder =
5195                         to_intel_encoder(connector_state->best_encoder);
5196
5197                 if (connector_state->crtc != &crtc->base)
5198                         continue;
5199
5200                 ret = encoder->compute_config(encoder, crtc_state,
5201                                               connector_state);
5202                 if (ret == -EDEADLK)
5203                         return ret;
5204                 if (ret < 0) {
5205                         drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] config failure: %d\n",
5206                                     encoder->base.base.id, encoder->base.name, ret);
5207                         return ret;
5208                 }
5209         }
5210
5211         /* Set default port clock if not overwritten by the encoder. Needs to be
5212          * done afterwards in case the encoder adjusts the mode. */
5213         if (!crtc_state->port_clock)
5214                 crtc_state->port_clock = crtc_state->hw.adjusted_mode.crtc_clock
5215                         * crtc_state->pixel_multiplier;
5216
5217         ret = intel_crtc_compute_config(state, crtc);
5218         if (ret == -EDEADLK)
5219                 return ret;
5220         if (ret == -EAGAIN) {
5221                 if (drm_WARN(&i915->drm, !retry,
5222                              "[CRTC:%d:%s] loop in pipe configuration computation\n",
5223                              crtc->base.base.id, crtc->base.name))
5224                         return -EINVAL;
5225
5226                 drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] bw constrained, retrying\n",
5227                             crtc->base.base.id, crtc->base.name);
5228                 retry = false;
5229                 goto encoder_retry;
5230         }
5231         if (ret < 0) {
5232                 drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] config failure: %d\n",
5233                             crtc->base.base.id, crtc->base.name, ret);
5234                 return ret;
5235         }
5236
5237         /* Dithering seems to not pass-through bits correctly when it should, so
5238          * only enable it on 6bpc panels and when its not a compliance
5239          * test requesting 6bpc video pattern.
5240          */
5241         crtc_state->dither = (crtc_state->pipe_bpp == 6*3) &&
5242                 !crtc_state->dither_force_disable;
5243         drm_dbg_kms(&i915->drm,
5244                     "[CRTC:%d:%s] hw max bpp: %i, pipe bpp: %i, dithering: %i\n",
5245                     crtc->base.base.id, crtc->base.name,
5246                     base_bpp, crtc_state->pipe_bpp, crtc_state->dither);
5247
5248         return 0;
5249 }
5250
5251 static int
5252 intel_modeset_pipe_config_late(struct intel_atomic_state *state,
5253                                struct intel_crtc *crtc)
5254 {
5255         struct intel_crtc_state *crtc_state =
5256                 intel_atomic_get_new_crtc_state(state, crtc);
5257         struct drm_connector_state *conn_state;
5258         struct drm_connector *connector;
5259         int i;
5260
5261         intel_bigjoiner_adjust_pipe_src(crtc_state);
5262
5263         for_each_new_connector_in_state(&state->base, connector,
5264                                         conn_state, i) {
5265                 struct intel_encoder *encoder =
5266                         to_intel_encoder(conn_state->best_encoder);
5267                 int ret;
5268
5269                 if (conn_state->crtc != &crtc->base ||
5270                     !encoder->compute_config_late)
5271                         continue;
5272
5273                 ret = encoder->compute_config_late(encoder, crtc_state,
5274                                                    conn_state);
5275                 if (ret)
5276                         return ret;
5277         }
5278
5279         return 0;
5280 }
5281
5282 bool intel_fuzzy_clock_check(int clock1, int clock2)
5283 {
5284         int diff;
5285
5286         if (clock1 == clock2)
5287                 return true;
5288
5289         if (!clock1 || !clock2)
5290                 return false;
5291
5292         diff = abs(clock1 - clock2);
5293
5294         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
5295                 return true;
5296
5297         return false;
5298 }
5299
5300 static bool
5301 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
5302                        const struct intel_link_m_n *m2_n2)
5303 {
5304         return m_n->tu == m2_n2->tu &&
5305                 m_n->data_m == m2_n2->data_m &&
5306                 m_n->data_n == m2_n2->data_n &&
5307                 m_n->link_m == m2_n2->link_m &&
5308                 m_n->link_n == m2_n2->link_n;
5309 }
5310
5311 static bool
5312 intel_compare_infoframe(const union hdmi_infoframe *a,
5313                         const union hdmi_infoframe *b)
5314 {
5315         return memcmp(a, b, sizeof(*a)) == 0;
5316 }
5317
5318 static bool
5319 intel_compare_dp_vsc_sdp(const struct drm_dp_vsc_sdp *a,
5320                          const struct drm_dp_vsc_sdp *b)
5321 {
5322         return memcmp(a, b, sizeof(*a)) == 0;
5323 }
5324
5325 static bool
5326 intel_compare_buffer(const u8 *a, const u8 *b, size_t len)
5327 {
5328         return memcmp(a, b, len) == 0;
5329 }
5330
5331 static void
5332 pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv,
5333                                bool fastset, const char *name,
5334                                const union hdmi_infoframe *a,
5335                                const union hdmi_infoframe *b)
5336 {
5337         if (fastset) {
5338                 if (!drm_debug_enabled(DRM_UT_KMS))
5339                         return;
5340
5341                 drm_dbg_kms(&dev_priv->drm,
5342                             "fastset mismatch in %s infoframe\n", name);
5343                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
5344                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, a);
5345                 drm_dbg_kms(&dev_priv->drm, "found:\n");
5346                 hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, b);
5347         } else {
5348                 drm_err(&dev_priv->drm, "mismatch in %s infoframe\n", name);
5349                 drm_err(&dev_priv->drm, "expected:\n");
5350                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, a);
5351                 drm_err(&dev_priv->drm, "found:\n");
5352                 hdmi_infoframe_log(KERN_ERR, dev_priv->drm.dev, b);
5353         }
5354 }
5355
5356 static void
5357 pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *dev_priv,
5358                                 bool fastset, const char *name,
5359                                 const struct drm_dp_vsc_sdp *a,
5360                                 const struct drm_dp_vsc_sdp *b)
5361 {
5362         if (fastset) {
5363                 if (!drm_debug_enabled(DRM_UT_KMS))
5364                         return;
5365
5366                 drm_dbg_kms(&dev_priv->drm,
5367                             "fastset mismatch in %s dp sdp\n", name);
5368                 drm_dbg_kms(&dev_priv->drm, "expected:\n");
5369                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, a);
5370                 drm_dbg_kms(&dev_priv->drm, "found:\n");
5371                 drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, b);
5372         } else {
5373                 drm_err(&dev_priv->drm, "mismatch in %s dp sdp\n", name);
5374                 drm_err(&dev_priv->drm, "expected:\n");
5375                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, a);
5376                 drm_err(&dev_priv->drm, "found:\n");
5377                 drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, b);
5378         }
5379 }
5380
5381 /* Returns the length up to and including the last differing byte */
5382 static size_t
5383 memcmp_diff_len(const u8 *a, const u8 *b, size_t len)
5384 {
5385         int i;
5386
5387         for (i = len - 1; i >= 0; i--) {
5388                 if (a[i] != b[i])
5389                         return i + 1;
5390         }
5391
5392         return 0;
5393 }
5394
5395 static void
5396 pipe_config_buffer_mismatch(struct drm_i915_private *dev_priv,
5397                             bool fastset, const char *name,
5398                             const u8 *a, const u8 *b, size_t len)
5399 {
5400         if (fastset) {
5401                 if (!drm_debug_enabled(DRM_UT_KMS))
5402                         return;
5403
5404                 /* only dump up to the last difference */
5405                 len = memcmp_diff_len(a, b, len);
5406
5407                 drm_dbg_kms(&dev_priv->drm,
5408                             "fastset mismatch in %s buffer\n", name);
5409                 print_hex_dump(KERN_DEBUG, "expected: ", DUMP_PREFIX_NONE,
5410                                16, 0, a, len, false);
5411                 print_hex_dump(KERN_DEBUG, "found: ", DUMP_PREFIX_NONE,
5412                                16, 0, b, len, false);
5413         } else {
5414                 /* only dump up to the last difference */
5415                 len = memcmp_diff_len(a, b, len);
5416
5417                 drm_err(&dev_priv->drm, "mismatch in %s buffer\n", name);
5418                 print_hex_dump(KERN_ERR, "expected: ", DUMP_PREFIX_NONE,
5419                                16, 0, a, len, false);
5420                 print_hex_dump(KERN_ERR, "found: ", DUMP_PREFIX_NONE,
5421                                16, 0, b, len, false);
5422         }
5423 }
5424
5425 static void __printf(4, 5)
5426 pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc,
5427                      const char *name, const char *format, ...)
5428 {
5429         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
5430         struct va_format vaf;
5431         va_list args;
5432
5433         va_start(args, format);
5434         vaf.fmt = format;
5435         vaf.va = &args;
5436
5437         if (fastset)
5438                 drm_dbg_kms(&i915->drm,
5439                             "[CRTC:%d:%s] fastset mismatch in %s %pV\n",
5440                             crtc->base.base.id, crtc->base.name, name, &vaf);
5441         else
5442                 drm_err(&i915->drm, "[CRTC:%d:%s] mismatch in %s %pV\n",
5443                         crtc->base.base.id, crtc->base.name, name, &vaf);
5444
5445         va_end(args);
5446 }
5447
5448 static bool fastboot_enabled(struct drm_i915_private *dev_priv)
5449 {
5450         if (dev_priv->params.fastboot != -1)
5451                 return dev_priv->params.fastboot;
5452
5453         /* Enable fastboot by default on Skylake and newer */
5454         if (DISPLAY_VER(dev_priv) >= 9)
5455                 return true;
5456
5457         /* Enable fastboot by default on VLV and CHV */
5458         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
5459                 return true;
5460
5461         /* Disabled by default on all others */
5462         return false;
5463 }
5464
5465 bool
5466 intel_pipe_config_compare(const struct intel_crtc_state *current_config,
5467                           const struct intel_crtc_state *pipe_config,
5468                           bool fastset)
5469 {
5470         struct drm_i915_private *dev_priv = to_i915(current_config->uapi.crtc->dev);
5471         struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
5472         bool ret = true;
5473         bool fixup_inherited = fastset &&
5474                 current_config->inherited && !pipe_config->inherited;
5475
5476         if (fixup_inherited && !fastboot_enabled(dev_priv)) {
5477                 drm_dbg_kms(&dev_priv->drm,
5478                             "initial modeset and fastboot not set\n");
5479                 ret = false;
5480         }
5481
5482 #define PIPE_CONF_CHECK_X(name) do { \
5483         if (current_config->name != pipe_config->name) { \
5484                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
5485                                      "(expected 0x%08x, found 0x%08x)", \
5486                                      current_config->name, \
5487                                      pipe_config->name); \
5488                 ret = false; \
5489         } \
5490 } while (0)
5491
5492 #define PIPE_CONF_CHECK_X_WITH_MASK(name, mask) do { \
5493         if ((current_config->name & (mask)) != (pipe_config->name & (mask))) { \
5494                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
5495                                      "(expected 0x%08x, found 0x%08x)", \
5496                                      current_config->name & (mask), \
5497                                      pipe_config->name & (mask)); \
5498                 ret = false; \
5499         } \
5500 } while (0)
5501
5502 #define PIPE_CONF_CHECK_I(name) do { \
5503         if (current_config->name != pipe_config->name) { \
5504                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
5505                                      "(expected %i, found %i)", \
5506                                      current_config->name, \
5507                                      pipe_config->name); \
5508                 ret = false; \
5509         } \
5510 } while (0)
5511
5512 #define PIPE_CONF_CHECK_BOOL(name) do { \
5513         if (current_config->name != pipe_config->name) { \
5514                 pipe_config_mismatch(fastset, crtc,  __stringify(name), \
5515                                      "(expected %s, found %s)", \
5516                                      str_yes_no(current_config->name), \
5517                                      str_yes_no(pipe_config->name)); \
5518                 ret = false; \
5519         } \
5520 } while (0)
5521
5522 /*
5523  * Checks state where we only read out the enabling, but not the entire
5524  * state itself (like full infoframes or ELD for audio). These states
5525  * require a full modeset on bootup to fix up.
5526  */
5527 #define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) do { \
5528         if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
5529                 PIPE_CONF_CHECK_BOOL(name); \
5530         } else { \
5531                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
5532                                      "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)", \
5533                                      str_yes_no(current_config->name), \
5534                                      str_yes_no(pipe_config->name)); \
5535                 ret = false; \
5536         } \
5537 } while (0)
5538
5539 #define PIPE_CONF_CHECK_P(name) do { \
5540         if (current_config->name != pipe_config->name) { \
5541                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
5542                                      "(expected %p, found %p)", \
5543                                      current_config->name, \
5544                                      pipe_config->name); \
5545                 ret = false; \
5546         } \
5547 } while (0)
5548
5549 #define PIPE_CONF_CHECK_M_N(name) do { \
5550         if (!intel_compare_link_m_n(&current_config->name, \
5551                                     &pipe_config->name)) { \
5552                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
5553                                      "(expected tu %i data %i/%i link %i/%i, " \
5554                                      "found tu %i, data %i/%i link %i/%i)", \
5555                                      current_config->name.tu, \
5556                                      current_config->name.data_m, \
5557                                      current_config->name.data_n, \
5558                                      current_config->name.link_m, \
5559                                      current_config->name.link_n, \
5560                                      pipe_config->name.tu, \
5561                                      pipe_config->name.data_m, \
5562                                      pipe_config->name.data_n, \
5563                                      pipe_config->name.link_m, \
5564                                      pipe_config->name.link_n); \
5565                 ret = false; \
5566         } \
5567 } while (0)
5568
5569 #define PIPE_CONF_CHECK_TIMINGS(name) do { \
5570         PIPE_CONF_CHECK_I(name.crtc_hdisplay); \
5571         PIPE_CONF_CHECK_I(name.crtc_htotal); \
5572         PIPE_CONF_CHECK_I(name.crtc_hblank_start); \
5573         PIPE_CONF_CHECK_I(name.crtc_hblank_end); \
5574         PIPE_CONF_CHECK_I(name.crtc_hsync_start); \
5575         PIPE_CONF_CHECK_I(name.crtc_hsync_end); \
5576         PIPE_CONF_CHECK_I(name.crtc_vdisplay); \
5577         PIPE_CONF_CHECK_I(name.crtc_vtotal); \
5578         PIPE_CONF_CHECK_I(name.crtc_vblank_start); \
5579         PIPE_CONF_CHECK_I(name.crtc_vblank_end); \
5580         PIPE_CONF_CHECK_I(name.crtc_vsync_start); \
5581         PIPE_CONF_CHECK_I(name.crtc_vsync_end); \
5582 } while (0)
5583
5584 #define PIPE_CONF_CHECK_RECT(name) do { \
5585         PIPE_CONF_CHECK_I(name.x1); \
5586         PIPE_CONF_CHECK_I(name.x2); \
5587         PIPE_CONF_CHECK_I(name.y1); \
5588         PIPE_CONF_CHECK_I(name.y2); \
5589 } while (0)
5590
5591 #define PIPE_CONF_CHECK_FLAGS(name, mask) do { \
5592         if ((current_config->name ^ pipe_config->name) & (mask)) { \
5593                 pipe_config_mismatch(fastset, crtc, __stringify(name), \
5594                                      "(%x) (expected %i, found %i)", \
5595                                      (mask), \
5596                                      current_config->name & (mask), \
5597                                      pipe_config->name & (mask)); \
5598                 ret = false; \
5599         } \
5600 } while (0)
5601
5602 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
5603         if (!intel_compare_infoframe(&current_config->infoframes.name, \
5604                                      &pipe_config->infoframes.name)) { \
5605                 pipe_config_infoframe_mismatch(dev_priv, fastset, __stringify(name), \
5606                                                &current_config->infoframes.name, \
5607                                                &pipe_config->infoframes.name); \
5608                 ret = false; \
5609         } \
5610 } while (0)
5611
5612 #define PIPE_CONF_CHECK_DP_VSC_SDP(name) do { \
5613         if (!current_config->has_psr && !pipe_config->has_psr && \
5614             !intel_compare_dp_vsc_sdp(&current_config->infoframes.name, \
5615                                       &pipe_config->infoframes.name)) { \
5616                 pipe_config_dp_vsc_sdp_mismatch(dev_priv, fastset, __stringify(name), \
5617                                                 &current_config->infoframes.name, \
5618                                                 &pipe_config->infoframes.name); \
5619                 ret = false; \
5620         } \
5621 } while (0)
5622
5623 #define PIPE_CONF_CHECK_BUFFER(name, len) do { \
5624         BUILD_BUG_ON(sizeof(current_config->name) != (len)); \
5625         BUILD_BUG_ON(sizeof(pipe_config->name) != (len)); \
5626         if (!intel_compare_buffer(current_config->name, pipe_config->name, (len))) { \
5627                 pipe_config_buffer_mismatch(dev_priv, fastset, __stringify(name), \
5628                                             current_config->name, \
5629                                             pipe_config->name, \
5630                                             (len)); \
5631                 ret = false; \
5632         } \
5633 } while (0)
5634
5635 #define PIPE_CONF_CHECK_COLOR_LUT(lut, is_pre_csc_lut) do { \
5636         if (current_config->gamma_mode == pipe_config->gamma_mode && \
5637             !intel_color_lut_equal(current_config, \
5638                                    current_config->lut, pipe_config->lut, \
5639                                    is_pre_csc_lut)) {   \
5640                 pipe_config_mismatch(fastset, crtc, __stringify(lut), \
5641                                      "hw_state doesn't match sw_state"); \
5642                 ret = false; \
5643         } \
5644 } while (0)
5645
5646 #define PIPE_CONF_QUIRK(quirk) \
5647         ((current_config->quirks | pipe_config->quirks) & (quirk))
5648
5649         PIPE_CONF_CHECK_I(hw.enable);
5650         PIPE_CONF_CHECK_I(hw.active);
5651
5652         PIPE_CONF_CHECK_I(cpu_transcoder);
5653         PIPE_CONF_CHECK_I(mst_master_transcoder);
5654
5655         PIPE_CONF_CHECK_BOOL(has_pch_encoder);
5656         PIPE_CONF_CHECK_I(fdi_lanes);
5657         PIPE_CONF_CHECK_M_N(fdi_m_n);
5658
5659         PIPE_CONF_CHECK_I(lane_count);
5660         PIPE_CONF_CHECK_X(lane_lat_optim_mask);
5661
5662         if (HAS_DOUBLE_BUFFERED_M_N(dev_priv)) {
5663                 if (!fastset || !pipe_config->seamless_m_n)
5664                         PIPE_CONF_CHECK_M_N(dp_m_n);
5665         } else {
5666                 PIPE_CONF_CHECK_M_N(dp_m_n);
5667                 PIPE_CONF_CHECK_M_N(dp_m2_n2);
5668         }
5669
5670         PIPE_CONF_CHECK_X(output_types);
5671
5672         PIPE_CONF_CHECK_I(framestart_delay);
5673         PIPE_CONF_CHECK_I(msa_timing_delay);
5674
5675         PIPE_CONF_CHECK_TIMINGS(hw.pipe_mode);
5676         PIPE_CONF_CHECK_TIMINGS(hw.adjusted_mode);
5677
5678         PIPE_CONF_CHECK_I(pixel_multiplier);
5679
5680         PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
5681                               DRM_MODE_FLAG_INTERLACE);
5682
5683         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
5684                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
5685                                       DRM_MODE_FLAG_PHSYNC);
5686                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
5687                                       DRM_MODE_FLAG_NHSYNC);
5688                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
5689                                       DRM_MODE_FLAG_PVSYNC);
5690                 PIPE_CONF_CHECK_FLAGS(hw.adjusted_mode.flags,
5691                                       DRM_MODE_FLAG_NVSYNC);
5692         }
5693
5694         PIPE_CONF_CHECK_I(output_format);
5695         PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
5696         if ((DISPLAY_VER(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
5697             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
5698                 PIPE_CONF_CHECK_BOOL(limited_color_range);
5699
5700         PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
5701         PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
5702         PIPE_CONF_CHECK_BOOL(has_infoframe);
5703         PIPE_CONF_CHECK_BOOL(fec_enable);
5704
5705         PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
5706         PIPE_CONF_CHECK_BUFFER(eld, MAX_ELD_BYTES);
5707
5708         PIPE_CONF_CHECK_X(gmch_pfit.control);
5709         /* pfit ratios are autocomputed by the hw on gen4+ */
5710         if (DISPLAY_VER(dev_priv) < 4)
5711                 PIPE_CONF_CHECK_X(gmch_pfit.pgm_ratios);
5712         PIPE_CONF_CHECK_X(gmch_pfit.lvds_border_bits);
5713
5714         /*
5715          * Changing the EDP transcoder input mux
5716          * (A_ONOFF vs. A_ON) requires a full modeset.
5717          */
5718         PIPE_CONF_CHECK_BOOL(pch_pfit.force_thru);
5719
5720         if (!fastset) {
5721                 PIPE_CONF_CHECK_RECT(pipe_src);
5722
5723                 PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
5724                 PIPE_CONF_CHECK_RECT(pch_pfit.dst);
5725
5726                 PIPE_CONF_CHECK_I(scaler_state.scaler_id);
5727                 PIPE_CONF_CHECK_I(pixel_rate);
5728
5729                 PIPE_CONF_CHECK_X(gamma_mode);
5730                 if (IS_CHERRYVIEW(dev_priv))
5731                         PIPE_CONF_CHECK_X(cgm_mode);
5732                 else
5733                         PIPE_CONF_CHECK_X(csc_mode);
5734                 PIPE_CONF_CHECK_BOOL(gamma_enable);
5735                 PIPE_CONF_CHECK_BOOL(csc_enable);
5736
5737                 PIPE_CONF_CHECK_I(linetime);
5738                 PIPE_CONF_CHECK_I(ips_linetime);
5739
5740                 PIPE_CONF_CHECK_COLOR_LUT(pre_csc_lut, true);
5741                 PIPE_CONF_CHECK_COLOR_LUT(post_csc_lut, false);
5742
5743                 if (current_config->active_planes) {
5744                         PIPE_CONF_CHECK_BOOL(has_psr);
5745                         PIPE_CONF_CHECK_BOOL(has_psr2);
5746                         PIPE_CONF_CHECK_BOOL(enable_psr2_sel_fetch);
5747                         PIPE_CONF_CHECK_I(dc3co_exitline);
5748                 }
5749         }
5750
5751         PIPE_CONF_CHECK_BOOL(double_wide);
5752
5753         if (dev_priv->display.dpll.mgr) {
5754                 PIPE_CONF_CHECK_P(shared_dpll);
5755
5756                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
5757                 PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
5758                 PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
5759                 PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
5760                 PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
5761                 PIPE_CONF_CHECK_X(dpll_hw_state.spll);
5762                 PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
5763                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
5764                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
5765                 PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr0);
5766                 PIPE_CONF_CHECK_X(dpll_hw_state.div0);
5767                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb0);
5768                 PIPE_CONF_CHECK_X(dpll_hw_state.ebb4);
5769                 PIPE_CONF_CHECK_X(dpll_hw_state.pll0);
5770                 PIPE_CONF_CHECK_X(dpll_hw_state.pll1);
5771                 PIPE_CONF_CHECK_X(dpll_hw_state.pll2);
5772                 PIPE_CONF_CHECK_X(dpll_hw_state.pll3);
5773                 PIPE_CONF_CHECK_X(dpll_hw_state.pll6);
5774                 PIPE_CONF_CHECK_X(dpll_hw_state.pll8);
5775                 PIPE_CONF_CHECK_X(dpll_hw_state.pll9);
5776                 PIPE_CONF_CHECK_X(dpll_hw_state.pll10);
5777                 PIPE_CONF_CHECK_X(dpll_hw_state.pcsdw12);
5778                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_refclkin_ctl);
5779                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_coreclkctl1);
5780                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_clktop2_hsclkctl);
5781                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div0);
5782                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_div1);
5783                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_lf);
5784                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_frac_lock);
5785                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_ssc);
5786                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_bias);
5787                 PIPE_CONF_CHECK_X(dpll_hw_state.mg_pll_tdc_coldst_bias);
5788         }
5789
5790         PIPE_CONF_CHECK_X(dsi_pll.ctrl);
5791         PIPE_CONF_CHECK_X(dsi_pll.div);
5792
5793         if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5)
5794                 PIPE_CONF_CHECK_I(pipe_bpp);
5795
5796         if (!fastset || !pipe_config->seamless_m_n) {
5797                 PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_clock);
5798                 PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_clock);
5799         }
5800         PIPE_CONF_CHECK_I(port_clock);
5801
5802         PIPE_CONF_CHECK_I(min_voltage_level);
5803
5804         if (current_config->has_psr || pipe_config->has_psr)
5805                 PIPE_CONF_CHECK_X_WITH_MASK(infoframes.enable,
5806                                             ~intel_hdmi_infoframe_enable(DP_SDP_VSC));
5807         else
5808                 PIPE_CONF_CHECK_X(infoframes.enable);
5809
5810         PIPE_CONF_CHECK_X(infoframes.gcp);
5811         PIPE_CONF_CHECK_INFOFRAME(avi);
5812         PIPE_CONF_CHECK_INFOFRAME(spd);
5813         PIPE_CONF_CHECK_INFOFRAME(hdmi);
5814         PIPE_CONF_CHECK_INFOFRAME(drm);
5815         PIPE_CONF_CHECK_DP_VSC_SDP(vsc);
5816
5817         PIPE_CONF_CHECK_X(sync_mode_slaves_mask);
5818         PIPE_CONF_CHECK_I(master_transcoder);
5819         PIPE_CONF_CHECK_X(bigjoiner_pipes);
5820
5821         PIPE_CONF_CHECK_I(dsc.compression_enable);
5822         PIPE_CONF_CHECK_I(dsc.dsc_split);
5823         PIPE_CONF_CHECK_I(dsc.compressed_bpp);
5824
5825         PIPE_CONF_CHECK_BOOL(splitter.enable);
5826         PIPE_CONF_CHECK_I(splitter.link_count);
5827         PIPE_CONF_CHECK_I(splitter.pixel_overlap);
5828
5829         PIPE_CONF_CHECK_BOOL(vrr.enable);
5830         PIPE_CONF_CHECK_I(vrr.vmin);
5831         PIPE_CONF_CHECK_I(vrr.vmax);
5832         PIPE_CONF_CHECK_I(vrr.flipline);
5833         PIPE_CONF_CHECK_I(vrr.pipeline_full);
5834         PIPE_CONF_CHECK_I(vrr.guardband);
5835
5836 #undef PIPE_CONF_CHECK_X
5837 #undef PIPE_CONF_CHECK_I
5838 #undef PIPE_CONF_CHECK_BOOL
5839 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
5840 #undef PIPE_CONF_CHECK_P
5841 #undef PIPE_CONF_CHECK_FLAGS
5842 #undef PIPE_CONF_CHECK_COLOR_LUT
5843 #undef PIPE_CONF_CHECK_TIMINGS
5844 #undef PIPE_CONF_CHECK_RECT
5845 #undef PIPE_CONF_QUIRK
5846
5847         return ret;
5848 }
5849
5850 static void
5851 intel_verify_planes(struct intel_atomic_state *state)
5852 {
5853         struct intel_plane *plane;
5854         const struct intel_plane_state *plane_state;
5855         int i;
5856
5857         for_each_new_intel_plane_in_state(state, plane,
5858                                           plane_state, i)
5859                 assert_plane(plane, plane_state->planar_slave ||
5860                              plane_state->uapi.visible);
5861 }
5862
5863 int intel_modeset_all_pipes(struct intel_atomic_state *state,
5864                             const char *reason)
5865 {
5866         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5867         struct intel_crtc *crtc;
5868
5869         /*
5870          * Add all pipes to the state, and force
5871          * a modeset on all the active ones.
5872          */
5873         for_each_intel_crtc(&dev_priv->drm, crtc) {
5874                 struct intel_crtc_state *crtc_state;
5875                 int ret;
5876
5877                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5878                 if (IS_ERR(crtc_state))
5879                         return PTR_ERR(crtc_state);
5880
5881                 if (!crtc_state->hw.active ||
5882                     intel_crtc_needs_modeset(crtc_state))
5883                         continue;
5884
5885                 drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] Full modeset due to %s\n",
5886                             crtc->base.base.id, crtc->base.name, reason);
5887
5888                 crtc_state->uapi.mode_changed = true;
5889                 crtc_state->update_pipe = false;
5890
5891                 ret = drm_atomic_add_affected_connectors(&state->base,
5892                                                          &crtc->base);
5893                 if (ret)
5894                         return ret;
5895
5896                 ret = intel_dp_mst_add_topology_state_for_crtc(state, crtc);
5897                 if (ret)
5898                         return ret;
5899
5900                 ret = intel_atomic_add_affected_planes(state, crtc);
5901                 if (ret)
5902                         return ret;
5903
5904                 crtc_state->update_planes |= crtc_state->active_planes;
5905                 crtc_state->async_flip_planes = 0;
5906                 crtc_state->do_async_flip = false;
5907         }
5908
5909         return 0;
5910 }
5911
5912 /*
5913  * This implements the workaround described in the "notes" section of the mode
5914  * set sequence documentation. When going from no pipes or single pipe to
5915  * multiple pipes, and planes are enabled after the pipe, we need to wait at
5916  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
5917  */
5918 static int hsw_mode_set_planes_workaround(struct intel_atomic_state *state)
5919 {
5920         struct intel_crtc_state *crtc_state;
5921         struct intel_crtc *crtc;
5922         struct intel_crtc_state *first_crtc_state = NULL;
5923         struct intel_crtc_state *other_crtc_state = NULL;
5924         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
5925         int i;
5926
5927         /* look at all crtc's that are going to be enabled in during modeset */
5928         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
5929                 if (!crtc_state->hw.active ||
5930                     !intel_crtc_needs_modeset(crtc_state))
5931                         continue;
5932
5933                 if (first_crtc_state) {
5934                         other_crtc_state = crtc_state;
5935                         break;
5936                 } else {
5937                         first_crtc_state = crtc_state;
5938                         first_pipe = crtc->pipe;
5939                 }
5940         }
5941
5942         /* No workaround needed? */
5943         if (!first_crtc_state)
5944                 return 0;
5945
5946         /* w/a possibly needed, check how many crtc's are already enabled. */
5947         for_each_intel_crtc(state->base.dev, crtc) {
5948                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
5949                 if (IS_ERR(crtc_state))
5950                         return PTR_ERR(crtc_state);
5951
5952                 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
5953
5954                 if (!crtc_state->hw.active ||
5955                     intel_crtc_needs_modeset(crtc_state))
5956                         continue;
5957
5958                 /* 2 or more enabled crtcs means no need for w/a */
5959                 if (enabled_pipe != INVALID_PIPE)
5960                         return 0;
5961
5962                 enabled_pipe = crtc->pipe;
5963         }
5964
5965         if (enabled_pipe != INVALID_PIPE)
5966                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
5967         else if (other_crtc_state)
5968                 other_crtc_state->hsw_workaround_pipe = first_pipe;
5969
5970         return 0;
5971 }
5972
5973 u8 intel_calc_active_pipes(struct intel_atomic_state *state,
5974                            u8 active_pipes)
5975 {
5976         const struct intel_crtc_state *crtc_state;
5977         struct intel_crtc *crtc;
5978         int i;
5979
5980         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
5981                 if (crtc_state->hw.active)
5982                         active_pipes |= BIT(crtc->pipe);
5983                 else
5984                         active_pipes &= ~BIT(crtc->pipe);
5985         }
5986
5987         return active_pipes;
5988 }
5989
5990 static int intel_modeset_checks(struct intel_atomic_state *state)
5991 {
5992         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
5993
5994         state->modeset = true;
5995
5996         if (IS_HASWELL(dev_priv))
5997                 return hsw_mode_set_planes_workaround(state);
5998
5999         return 0;
6000 }
6001
6002 static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_state,
6003                                      struct intel_crtc_state *new_crtc_state)
6004 {
6005         if (!intel_pipe_config_compare(old_crtc_state, new_crtc_state, true))
6006                 return;
6007
6008         new_crtc_state->uapi.mode_changed = false;
6009         if (!intel_crtc_needs_modeset(new_crtc_state))
6010                 new_crtc_state->update_pipe = true;
6011 }
6012
6013 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
6014                                           struct intel_crtc *crtc,
6015                                           u8 plane_ids_mask)
6016 {
6017         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6018         struct intel_plane *plane;
6019
6020         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6021                 struct intel_plane_state *plane_state;
6022
6023                 if ((plane_ids_mask & BIT(plane->id)) == 0)
6024                         continue;
6025
6026                 plane_state = intel_atomic_get_plane_state(state, plane);
6027                 if (IS_ERR(plane_state))
6028                         return PTR_ERR(plane_state);
6029         }
6030
6031         return 0;
6032 }
6033
6034 int intel_atomic_add_affected_planes(struct intel_atomic_state *state,
6035                                      struct intel_crtc *crtc)
6036 {
6037         const struct intel_crtc_state *old_crtc_state =
6038                 intel_atomic_get_old_crtc_state(state, crtc);
6039         const struct intel_crtc_state *new_crtc_state =
6040                 intel_atomic_get_new_crtc_state(state, crtc);
6041
6042         return intel_crtc_add_planes_to_state(state, crtc,
6043                                               old_crtc_state->enabled_planes |
6044                                               new_crtc_state->enabled_planes);
6045 }
6046
6047 static bool active_planes_affects_min_cdclk(struct drm_i915_private *dev_priv)
6048 {
6049         /* See {hsw,vlv,ivb}_plane_ratio() */
6050         return IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv) ||
6051                 IS_CHERRYVIEW(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
6052                 IS_IVYBRIDGE(dev_priv);
6053 }
6054
6055 static int intel_crtc_add_bigjoiner_planes(struct intel_atomic_state *state,
6056                                            struct intel_crtc *crtc,
6057                                            struct intel_crtc *other)
6058 {
6059         const struct intel_plane_state *plane_state;
6060         struct intel_plane *plane;
6061         u8 plane_ids = 0;
6062         int i;
6063
6064         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6065                 if (plane->pipe == crtc->pipe)
6066                         plane_ids |= BIT(plane->id);
6067         }
6068
6069         return intel_crtc_add_planes_to_state(state, other, plane_ids);
6070 }
6071
6072 static int intel_bigjoiner_add_affected_planes(struct intel_atomic_state *state)
6073 {
6074         struct drm_i915_private *i915 = to_i915(state->base.dev);
6075         const struct intel_crtc_state *crtc_state;
6076         struct intel_crtc *crtc;
6077         int i;
6078
6079         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6080                 struct intel_crtc *other;
6081
6082                 for_each_intel_crtc_in_pipe_mask(&i915->drm, other,
6083                                                  crtc_state->bigjoiner_pipes) {
6084                         int ret;
6085
6086                         if (crtc == other)
6087                                 continue;
6088
6089                         ret = intel_crtc_add_bigjoiner_planes(state, crtc, other);
6090                         if (ret)
6091                                 return ret;
6092                 }
6093         }
6094
6095         return 0;
6096 }
6097
6098 static int intel_atomic_check_planes(struct intel_atomic_state *state)
6099 {
6100         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6101         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
6102         struct intel_plane_state *plane_state;
6103         struct intel_plane *plane;
6104         struct intel_crtc *crtc;
6105         int i, ret;
6106
6107         ret = icl_add_linked_planes(state);
6108         if (ret)
6109                 return ret;
6110
6111         ret = intel_bigjoiner_add_affected_planes(state);
6112         if (ret)
6113                 return ret;
6114
6115         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
6116                 ret = intel_plane_atomic_check(state, plane);
6117                 if (ret) {
6118                         drm_dbg_atomic(&dev_priv->drm,
6119                                        "[PLANE:%d:%s] atomic driver check failed\n",
6120                                        plane->base.base.id, plane->base.name);
6121                         return ret;
6122                 }
6123         }
6124
6125         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6126                                             new_crtc_state, i) {
6127                 u8 old_active_planes, new_active_planes;
6128
6129                 ret = icl_check_nv12_planes(new_crtc_state);
6130                 if (ret)
6131                         return ret;
6132
6133                 /*
6134                  * On some platforms the number of active planes affects
6135                  * the planes' minimum cdclk calculation. Add such planes
6136                  * to the state before we compute the minimum cdclk.
6137                  */
6138                 if (!active_planes_affects_min_cdclk(dev_priv))
6139                         continue;
6140
6141                 old_active_planes = old_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
6142                 new_active_planes = new_crtc_state->active_planes & ~BIT(PLANE_CURSOR);
6143
6144                 if (hweight8(old_active_planes) == hweight8(new_active_planes))
6145                         continue;
6146
6147                 ret = intel_crtc_add_planes_to_state(state, crtc, new_active_planes);
6148                 if (ret)
6149                         return ret;
6150         }
6151
6152         return 0;
6153 }
6154
6155 static int intel_atomic_check_crtcs(struct intel_atomic_state *state)
6156 {
6157         struct intel_crtc_state *crtc_state;
6158         struct intel_crtc *crtc;
6159         int i;
6160
6161         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6162                 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
6163                 int ret;
6164
6165                 ret = intel_crtc_atomic_check(state, crtc);
6166                 if (ret) {
6167                         drm_dbg_atomic(&i915->drm,
6168                                        "[CRTC:%d:%s] atomic driver check failed\n",
6169                                        crtc->base.base.id, crtc->base.name);
6170                         return ret;
6171                 }
6172         }
6173
6174         return 0;
6175 }
6176
6177 static bool intel_cpu_transcoders_need_modeset(struct intel_atomic_state *state,
6178                                                u8 transcoders)
6179 {
6180         const struct intel_crtc_state *new_crtc_state;
6181         struct intel_crtc *crtc;
6182         int i;
6183
6184         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6185                 if (new_crtc_state->hw.enable &&
6186                     transcoders & BIT(new_crtc_state->cpu_transcoder) &&
6187                     intel_crtc_needs_modeset(new_crtc_state))
6188                         return true;
6189         }
6190
6191         return false;
6192 }
6193
6194 static bool intel_pipes_need_modeset(struct intel_atomic_state *state,
6195                                      u8 pipes)
6196 {
6197         const struct intel_crtc_state *new_crtc_state;
6198         struct intel_crtc *crtc;
6199         int i;
6200
6201         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6202                 if (new_crtc_state->hw.enable &&
6203                     pipes & BIT(crtc->pipe) &&
6204                     intel_crtc_needs_modeset(new_crtc_state))
6205                         return true;
6206         }
6207
6208         return false;
6209 }
6210
6211 static int intel_atomic_check_bigjoiner(struct intel_atomic_state *state,
6212                                         struct intel_crtc *master_crtc)
6213 {
6214         struct drm_i915_private *i915 = to_i915(state->base.dev);
6215         struct intel_crtc_state *master_crtc_state =
6216                 intel_atomic_get_new_crtc_state(state, master_crtc);
6217         struct intel_crtc *slave_crtc;
6218
6219         if (!master_crtc_state->bigjoiner_pipes)
6220                 return 0;
6221
6222         /* sanity check */
6223         if (drm_WARN_ON(&i915->drm,
6224                         master_crtc->pipe != bigjoiner_master_pipe(master_crtc_state)))
6225                 return -EINVAL;
6226
6227         if (master_crtc_state->bigjoiner_pipes & ~bigjoiner_pipes(i915)) {
6228                 drm_dbg_kms(&i915->drm,
6229                             "[CRTC:%d:%s] Cannot act as big joiner master "
6230                             "(need 0x%x as pipes, only 0x%x possible)\n",
6231                             master_crtc->base.base.id, master_crtc->base.name,
6232                             master_crtc_state->bigjoiner_pipes, bigjoiner_pipes(i915));
6233                 return -EINVAL;
6234         }
6235
6236         for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
6237                                          intel_crtc_bigjoiner_slave_pipes(master_crtc_state)) {
6238                 struct intel_crtc_state *slave_crtc_state;
6239                 int ret;
6240
6241                 slave_crtc_state = intel_atomic_get_crtc_state(&state->base, slave_crtc);
6242                 if (IS_ERR(slave_crtc_state))
6243                         return PTR_ERR(slave_crtc_state);
6244
6245                 /* master being enabled, slave was already configured? */
6246                 if (slave_crtc_state->uapi.enable) {
6247                         drm_dbg_kms(&i915->drm,
6248                                     "[CRTC:%d:%s] Slave is enabled as normal CRTC, but "
6249                                     "[CRTC:%d:%s] claiming this CRTC for bigjoiner.\n",
6250                                     slave_crtc->base.base.id, slave_crtc->base.name,
6251                                     master_crtc->base.base.id, master_crtc->base.name);
6252                         return -EINVAL;
6253                 }
6254
6255                 /*
6256                  * The state copy logic assumes the master crtc gets processed
6257                  * before the slave crtc during the main compute_config loop.
6258                  * This works because the crtcs are created in pipe order,
6259                  * and the hardware requires master pipe < slave pipe as well.
6260                  * Should that change we need to rethink the logic.
6261                  */
6262                 if (WARN_ON(drm_crtc_index(&master_crtc->base) >
6263                             drm_crtc_index(&slave_crtc->base)))
6264                         return -EINVAL;
6265
6266                 drm_dbg_kms(&i915->drm,
6267                             "[CRTC:%d:%s] Used as slave for big joiner master [CRTC:%d:%s]\n",
6268                             slave_crtc->base.base.id, slave_crtc->base.name,
6269                             master_crtc->base.base.id, master_crtc->base.name);
6270
6271                 slave_crtc_state->bigjoiner_pipes =
6272                         master_crtc_state->bigjoiner_pipes;
6273
6274                 ret = copy_bigjoiner_crtc_state_modeset(state, slave_crtc);
6275                 if (ret)
6276                         return ret;
6277         }
6278
6279         return 0;
6280 }
6281
6282 static void kill_bigjoiner_slave(struct intel_atomic_state *state,
6283                                  struct intel_crtc *master_crtc)
6284 {
6285         struct drm_i915_private *i915 = to_i915(state->base.dev);
6286         struct intel_crtc_state *master_crtc_state =
6287                 intel_atomic_get_new_crtc_state(state, master_crtc);
6288         struct intel_crtc *slave_crtc;
6289
6290         for_each_intel_crtc_in_pipe_mask(&i915->drm, slave_crtc,
6291                                          intel_crtc_bigjoiner_slave_pipes(master_crtc_state)) {
6292                 struct intel_crtc_state *slave_crtc_state =
6293                         intel_atomic_get_new_crtc_state(state, slave_crtc);
6294
6295                 slave_crtc_state->bigjoiner_pipes = 0;
6296
6297                 intel_crtc_copy_uapi_to_hw_state_modeset(state, slave_crtc);
6298         }
6299
6300         master_crtc_state->bigjoiner_pipes = 0;
6301 }
6302
6303 /**
6304  * DOC: asynchronous flip implementation
6305  *
6306  * Asynchronous page flip is the implementation for the DRM_MODE_PAGE_FLIP_ASYNC
6307  * flag. Currently async flip is only supported via the drmModePageFlip IOCTL.
6308  * Correspondingly, support is currently added for primary plane only.
6309  *
6310  * Async flip can only change the plane surface address, so anything else
6311  * changing is rejected from the intel_async_flip_check_hw() function.
6312  * Once this check is cleared, flip done interrupt is enabled using
6313  * the intel_crtc_enable_flip_done() function.
6314  *
6315  * As soon as the surface address register is written, flip done interrupt is
6316  * generated and the requested events are sent to the usersapce in the interrupt
6317  * handler itself. The timestamp and sequence sent during the flip done event
6318  * correspond to the last vblank and have no relation to the actual time when
6319  * the flip done event was sent.
6320  */
6321 static int intel_async_flip_check_uapi(struct intel_atomic_state *state,
6322                                        struct intel_crtc *crtc)
6323 {
6324         struct drm_i915_private *i915 = to_i915(state->base.dev);
6325         const struct intel_crtc_state *new_crtc_state =
6326                 intel_atomic_get_new_crtc_state(state, crtc);
6327         const struct intel_plane_state *old_plane_state;
6328         struct intel_plane_state *new_plane_state;
6329         struct intel_plane *plane;
6330         int i;
6331
6332         if (!new_crtc_state->uapi.async_flip)
6333                 return 0;
6334
6335         if (!new_crtc_state->uapi.active) {
6336                 drm_dbg_kms(&i915->drm,
6337                             "[CRTC:%d:%s] not active\n",
6338                             crtc->base.base.id, crtc->base.name);
6339                 return -EINVAL;
6340         }
6341
6342         if (intel_crtc_needs_modeset(new_crtc_state)) {
6343                 drm_dbg_kms(&i915->drm,
6344                             "[CRTC:%d:%s] modeset required\n",
6345                             crtc->base.base.id, crtc->base.name);
6346                 return -EINVAL;
6347         }
6348
6349         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
6350                                              new_plane_state, i) {
6351                 if (plane->pipe != crtc->pipe)
6352                         continue;
6353
6354                 /*
6355                  * TODO: Async flip is only supported through the page flip IOCTL
6356                  * as of now. So support currently added for primary plane only.
6357                  * Support for other planes on platforms on which supports
6358                  * this(vlv/chv and icl+) should be added when async flip is
6359                  * enabled in the atomic IOCTL path.
6360                  */
6361                 if (!plane->async_flip) {
6362                         drm_dbg_kms(&i915->drm,
6363                                     "[PLANE:%d:%s] async flip not supported\n",
6364                                     plane->base.base.id, plane->base.name);
6365                         return -EINVAL;
6366                 }
6367
6368                 if (!old_plane_state->uapi.fb || !new_plane_state->uapi.fb) {
6369                         drm_dbg_kms(&i915->drm,
6370                                     "[PLANE:%d:%s] no old or new framebuffer\n",
6371                                     plane->base.base.id, plane->base.name);
6372                         return -EINVAL;
6373                 }
6374         }
6375
6376         return 0;
6377 }
6378
6379 static int intel_async_flip_check_hw(struct intel_atomic_state *state, struct intel_crtc *crtc)
6380 {
6381         struct drm_i915_private *i915 = to_i915(state->base.dev);
6382         const struct intel_crtc_state *old_crtc_state, *new_crtc_state;
6383         const struct intel_plane_state *new_plane_state, *old_plane_state;
6384         struct intel_plane *plane;
6385         int i;
6386
6387         old_crtc_state = intel_atomic_get_old_crtc_state(state, crtc);
6388         new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
6389
6390         if (!new_crtc_state->uapi.async_flip)
6391                 return 0;
6392
6393         if (!new_crtc_state->hw.active) {
6394                 drm_dbg_kms(&i915->drm,
6395                             "[CRTC:%d:%s] not active\n",
6396                             crtc->base.base.id, crtc->base.name);
6397                 return -EINVAL;
6398         }
6399
6400         if (intel_crtc_needs_modeset(new_crtc_state)) {
6401                 drm_dbg_kms(&i915->drm,
6402                             "[CRTC:%d:%s] modeset required\n",
6403                             crtc->base.base.id, crtc->base.name);
6404                 return -EINVAL;
6405         }
6406
6407         if (old_crtc_state->active_planes != new_crtc_state->active_planes) {
6408                 drm_dbg_kms(&i915->drm,
6409                             "[CRTC:%d:%s] Active planes cannot be in async flip\n",
6410                             crtc->base.base.id, crtc->base.name);
6411                 return -EINVAL;
6412         }
6413
6414         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
6415                                              new_plane_state, i) {
6416                 if (plane->pipe != crtc->pipe)
6417                         continue;
6418
6419                 /*
6420                  * Only async flip capable planes should be in the state
6421                  * if we're really about to ask the hardware to perform
6422                  * an async flip. We should never get this far otherwise.
6423                  */
6424                 if (drm_WARN_ON(&i915->drm,
6425                                 new_crtc_state->do_async_flip && !plane->async_flip))
6426                         return -EINVAL;
6427
6428                 /*
6429                  * Only check async flip capable planes other planes
6430                  * may be involved in the initial commit due to
6431                  * the wm0/ddb optimization.
6432                  *
6433                  * TODO maybe should track which planes actually
6434                  * were requested to do the async flip...
6435                  */
6436                 if (!plane->async_flip)
6437                         continue;
6438
6439                 /*
6440                  * FIXME: This check is kept generic for all platforms.
6441                  * Need to verify this for all gen9 platforms to enable
6442                  * this selectively if required.
6443                  */
6444                 switch (new_plane_state->hw.fb->modifier) {
6445                 case I915_FORMAT_MOD_X_TILED:
6446                 case I915_FORMAT_MOD_Y_TILED:
6447                 case I915_FORMAT_MOD_Yf_TILED:
6448                 case I915_FORMAT_MOD_4_TILED:
6449                         break;
6450                 default:
6451                         drm_dbg_kms(&i915->drm,
6452                                     "[PLANE:%d:%s] Modifier does not support async flips\n",
6453                                     plane->base.base.id, plane->base.name);
6454                         return -EINVAL;
6455                 }
6456
6457                 if (new_plane_state->hw.fb->format->num_planes > 1) {
6458                         drm_dbg_kms(&i915->drm,
6459                                     "[PLANE:%d:%s] Planar formats do not support async flips\n",
6460                                     plane->base.base.id, plane->base.name);
6461                         return -EINVAL;
6462                 }
6463
6464                 if (old_plane_state->view.color_plane[0].mapping_stride !=
6465                     new_plane_state->view.color_plane[0].mapping_stride) {
6466                         drm_dbg_kms(&i915->drm,
6467                                     "[PLANE:%d:%s] Stride cannot be changed in async flip\n",
6468                                     plane->base.base.id, plane->base.name);
6469                         return -EINVAL;
6470                 }
6471
6472                 if (old_plane_state->hw.fb->modifier !=
6473                     new_plane_state->hw.fb->modifier) {
6474                         drm_dbg_kms(&i915->drm,
6475                                     "[PLANE:%d:%s] Modifier cannot be changed in async flip\n",
6476                                     plane->base.base.id, plane->base.name);
6477                         return -EINVAL;
6478                 }
6479
6480                 if (old_plane_state->hw.fb->format !=
6481                     new_plane_state->hw.fb->format) {
6482                         drm_dbg_kms(&i915->drm,
6483                                     "[PLANE:%d:%s] Pixel format cannot be changed in async flip\n",
6484                                     plane->base.base.id, plane->base.name);
6485                         return -EINVAL;
6486                 }
6487
6488                 if (old_plane_state->hw.rotation !=
6489                     new_plane_state->hw.rotation) {
6490                         drm_dbg_kms(&i915->drm,
6491                                     "[PLANE:%d:%s] Rotation cannot be changed in async flip\n",
6492                                     plane->base.base.id, plane->base.name);
6493                         return -EINVAL;
6494                 }
6495
6496                 if (!drm_rect_equals(&old_plane_state->uapi.src, &new_plane_state->uapi.src) ||
6497                     !drm_rect_equals(&old_plane_state->uapi.dst, &new_plane_state->uapi.dst)) {
6498                         drm_dbg_kms(&i915->drm,
6499                                     "[PLANE:%d:%s] Size/co-ordinates cannot be changed in async flip\n",
6500                                     plane->base.base.id, plane->base.name);
6501                         return -EINVAL;
6502                 }
6503
6504                 if (old_plane_state->hw.alpha != new_plane_state->hw.alpha) {
6505                         drm_dbg_kms(&i915->drm,
6506                                     "[PLANES:%d:%s] Alpha value cannot be changed in async flip\n",
6507                                     plane->base.base.id, plane->base.name);
6508                         return -EINVAL;
6509                 }
6510
6511                 if (old_plane_state->hw.pixel_blend_mode !=
6512                     new_plane_state->hw.pixel_blend_mode) {
6513                         drm_dbg_kms(&i915->drm,
6514                                     "[PLANE:%d:%s] Pixel blend mode cannot be changed in async flip\n",
6515                                     plane->base.base.id, plane->base.name);
6516                         return -EINVAL;
6517                 }
6518
6519                 if (old_plane_state->hw.color_encoding != new_plane_state->hw.color_encoding) {
6520                         drm_dbg_kms(&i915->drm,
6521                                     "[PLANE:%d:%s] Color encoding cannot be changed in async flip\n",
6522                                     plane->base.base.id, plane->base.name);
6523                         return -EINVAL;
6524                 }
6525
6526                 if (old_plane_state->hw.color_range != new_plane_state->hw.color_range) {
6527                         drm_dbg_kms(&i915->drm,
6528                                     "[PLANE:%d:%s] Color range cannot be changed in async flip\n",
6529                                     plane->base.base.id, plane->base.name);
6530                         return -EINVAL;
6531                 }
6532
6533                 /* plane decryption is allow to change only in synchronous flips */
6534                 if (old_plane_state->decrypt != new_plane_state->decrypt) {
6535                         drm_dbg_kms(&i915->drm,
6536                                     "[PLANE:%d:%s] Decryption cannot be changed in async flip\n",
6537                                     plane->base.base.id, plane->base.name);
6538                         return -EINVAL;
6539                 }
6540         }
6541
6542         return 0;
6543 }
6544
6545 static int intel_bigjoiner_add_affected_crtcs(struct intel_atomic_state *state)
6546 {
6547         struct drm_i915_private *i915 = to_i915(state->base.dev);
6548         struct intel_crtc_state *crtc_state;
6549         struct intel_crtc *crtc;
6550         u8 affected_pipes = 0;
6551         u8 modeset_pipes = 0;
6552         int i;
6553
6554         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6555                 affected_pipes |= crtc_state->bigjoiner_pipes;
6556                 if (intel_crtc_needs_modeset(crtc_state))
6557                         modeset_pipes |= crtc_state->bigjoiner_pipes;
6558         }
6559
6560         for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, affected_pipes) {
6561                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
6562                 if (IS_ERR(crtc_state))
6563                         return PTR_ERR(crtc_state);
6564         }
6565
6566         for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, modeset_pipes) {
6567                 int ret;
6568
6569                 crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
6570
6571                 crtc_state->uapi.mode_changed = true;
6572
6573                 ret = drm_atomic_add_affected_connectors(&state->base, &crtc->base);
6574                 if (ret)
6575                         return ret;
6576
6577                 ret = intel_atomic_add_affected_planes(state, crtc);
6578                 if (ret)
6579                         return ret;
6580         }
6581
6582         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6583                 /* Kill old bigjoiner link, we may re-establish afterwards */
6584                 if (intel_crtc_needs_modeset(crtc_state) &&
6585                     intel_crtc_is_bigjoiner_master(crtc_state))
6586                         kill_bigjoiner_slave(state, crtc);
6587         }
6588
6589         return 0;
6590 }
6591
6592 /**
6593  * intel_atomic_check - validate state object
6594  * @dev: drm device
6595  * @_state: state to validate
6596  */
6597 int intel_atomic_check(struct drm_device *dev,
6598                        struct drm_atomic_state *_state)
6599 {
6600         struct drm_i915_private *dev_priv = to_i915(dev);
6601         struct intel_atomic_state *state = to_intel_atomic_state(_state);
6602         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
6603         struct intel_crtc *crtc;
6604         int ret, i;
6605         bool any_ms = false;
6606
6607         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6608                                             new_crtc_state, i) {
6609                 if (new_crtc_state->inherited != old_crtc_state->inherited)
6610                         new_crtc_state->uapi.mode_changed = true;
6611
6612                 if (new_crtc_state->uapi.scaling_filter !=
6613                     old_crtc_state->uapi.scaling_filter)
6614                         new_crtc_state->uapi.mode_changed = true;
6615         }
6616
6617         intel_vrr_check_modeset(state);
6618
6619         ret = drm_atomic_helper_check_modeset(dev, &state->base);
6620         if (ret)
6621                 goto fail;
6622
6623         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6624                 ret = intel_async_flip_check_uapi(state, crtc);
6625                 if (ret)
6626                         return ret;
6627         }
6628
6629         ret = intel_bigjoiner_add_affected_crtcs(state);
6630         if (ret)
6631                 goto fail;
6632
6633         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6634                                             new_crtc_state, i) {
6635                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
6636                         if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
6637                                 copy_bigjoiner_crtc_state_nomodeset(state, crtc);
6638                         else
6639                                 intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc);
6640                         continue;
6641                 }
6642
6643                 if (intel_crtc_is_bigjoiner_slave(new_crtc_state)) {
6644                         drm_WARN_ON(&dev_priv->drm, new_crtc_state->uapi.enable);
6645                         continue;
6646                 }
6647
6648                 ret = intel_crtc_prepare_cleared_state(state, crtc);
6649                 if (ret)
6650                         goto fail;
6651
6652                 if (!new_crtc_state->hw.enable)
6653                         continue;
6654
6655                 ret = intel_modeset_pipe_config(state, crtc);
6656                 if (ret)
6657                         goto fail;
6658
6659                 ret = intel_atomic_check_bigjoiner(state, crtc);
6660                 if (ret)
6661                         goto fail;
6662         }
6663
6664         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6665                                             new_crtc_state, i) {
6666                 if (!intel_crtc_needs_modeset(new_crtc_state))
6667                         continue;
6668
6669                 if (new_crtc_state->hw.enable) {
6670                         ret = intel_modeset_pipe_config_late(state, crtc);
6671                         if (ret)
6672                                 goto fail;
6673                 }
6674
6675                 intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
6676         }
6677
6678         /**
6679          * Check if fastset is allowed by external dependencies like other
6680          * pipes and transcoders.
6681          *
6682          * Right now it only forces a fullmodeset when the MST master
6683          * transcoder did not changed but the pipe of the master transcoder
6684          * needs a fullmodeset so all slaves also needs to do a fullmodeset or
6685          * in case of port synced crtcs, if one of the synced crtcs
6686          * needs a full modeset, all other synced crtcs should be
6687          * forced a full modeset.
6688          */
6689         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6690                 if (!new_crtc_state->hw.enable || intel_crtc_needs_modeset(new_crtc_state))
6691                         continue;
6692
6693                 if (intel_dp_mst_is_slave_trans(new_crtc_state)) {
6694                         enum transcoder master = new_crtc_state->mst_master_transcoder;
6695
6696                         if (intel_cpu_transcoders_need_modeset(state, BIT(master))) {
6697                                 new_crtc_state->uapi.mode_changed = true;
6698                                 new_crtc_state->update_pipe = false;
6699                         }
6700                 }
6701
6702                 if (is_trans_port_sync_mode(new_crtc_state)) {
6703                         u8 trans = new_crtc_state->sync_mode_slaves_mask;
6704
6705                         if (new_crtc_state->master_transcoder != INVALID_TRANSCODER)
6706                                 trans |= BIT(new_crtc_state->master_transcoder);
6707
6708                         if (intel_cpu_transcoders_need_modeset(state, trans)) {
6709                                 new_crtc_state->uapi.mode_changed = true;
6710                                 new_crtc_state->update_pipe = false;
6711                         }
6712                 }
6713
6714                 if (new_crtc_state->bigjoiner_pipes) {
6715                         if (intel_pipes_need_modeset(state, new_crtc_state->bigjoiner_pipes)) {
6716                                 new_crtc_state->uapi.mode_changed = true;
6717                                 new_crtc_state->update_pipe = false;
6718                         }
6719                 }
6720         }
6721
6722         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6723                                             new_crtc_state, i) {
6724                 if (!intel_crtc_needs_modeset(new_crtc_state))
6725                         continue;
6726
6727                 any_ms = true;
6728
6729                 intel_release_shared_dplls(state, crtc);
6730         }
6731
6732         if (any_ms && !check_digital_port_conflicts(state)) {
6733                 drm_dbg_kms(&dev_priv->drm,
6734                             "rejecting conflicting digital port configuration\n");
6735                 ret = -EINVAL;
6736                 goto fail;
6737         }
6738
6739         ret = drm_dp_mst_atomic_check(&state->base);
6740         if (ret)
6741                 goto fail;
6742
6743         ret = intel_atomic_check_planes(state);
6744         if (ret)
6745                 goto fail;
6746
6747         ret = intel_compute_global_watermarks(state);
6748         if (ret)
6749                 goto fail;
6750
6751         ret = intel_bw_atomic_check(state);
6752         if (ret)
6753                 goto fail;
6754
6755         ret = intel_cdclk_atomic_check(state, &any_ms);
6756         if (ret)
6757                 goto fail;
6758
6759         if (intel_any_crtc_needs_modeset(state))
6760                 any_ms = true;
6761
6762         if (any_ms) {
6763                 ret = intel_modeset_checks(state);
6764                 if (ret)
6765                         goto fail;
6766
6767                 ret = intel_modeset_calc_cdclk(state);
6768                 if (ret)
6769                         return ret;
6770         }
6771
6772         ret = intel_atomic_check_crtcs(state);
6773         if (ret)
6774                 goto fail;
6775
6776         ret = intel_fbc_atomic_check(state);
6777         if (ret)
6778                 goto fail;
6779
6780         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6781                                             new_crtc_state, i) {
6782                 intel_color_assert_luts(new_crtc_state);
6783
6784                 ret = intel_async_flip_check_hw(state, crtc);
6785                 if (ret)
6786                         goto fail;
6787
6788                 /* Either full modeset or fastset (or neither), never both */
6789                 drm_WARN_ON(&dev_priv->drm,
6790                             intel_crtc_needs_modeset(new_crtc_state) &&
6791                             intel_crtc_needs_fastset(new_crtc_state));
6792
6793                 if (!intel_crtc_needs_modeset(new_crtc_state) &&
6794                     !intel_crtc_needs_fastset(new_crtc_state))
6795                         continue;
6796
6797                 intel_crtc_state_dump(new_crtc_state, state,
6798                                       intel_crtc_needs_modeset(new_crtc_state) ?
6799                                       "modeset" : "fastset");
6800         }
6801
6802         return 0;
6803
6804  fail:
6805         if (ret == -EDEADLK)
6806                 return ret;
6807
6808         /*
6809          * FIXME would probably be nice to know which crtc specifically
6810          * caused the failure, in cases where we can pinpoint it.
6811          */
6812         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6813                                             new_crtc_state, i)
6814                 intel_crtc_state_dump(new_crtc_state, state, "failed");
6815
6816         return ret;
6817 }
6818
6819 static int intel_atomic_prepare_commit(struct intel_atomic_state *state)
6820 {
6821         struct intel_crtc_state *crtc_state;
6822         struct intel_crtc *crtc;
6823         int i, ret;
6824
6825         ret = drm_atomic_helper_prepare_planes(state->base.dev, &state->base);
6826         if (ret < 0)
6827                 return ret;
6828
6829         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
6830                 if (intel_crtc_needs_color_update(crtc_state))
6831                         intel_color_prepare_commit(crtc_state);
6832         }
6833
6834         return 0;
6835 }
6836
6837 void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc,
6838                                   struct intel_crtc_state *crtc_state)
6839 {
6840         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6841
6842         if (DISPLAY_VER(dev_priv) != 2 || crtc_state->active_planes)
6843                 intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
6844
6845         if (crtc_state->has_pch_encoder) {
6846                 enum pipe pch_transcoder =
6847                         intel_crtc_pch_transcoder(crtc);
6848
6849                 intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
6850         }
6851 }
6852
6853 static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
6854                                const struct intel_crtc_state *new_crtc_state)
6855 {
6856         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6857         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6858
6859         /*
6860          * Update pipe size and adjust fitter if needed: the reason for this is
6861          * that in compute_mode_changes we check the native mode (not the pfit
6862          * mode) to see if we can flip rather than do a full mode set. In the
6863          * fastboot case, we'll flip, but if we don't update the pipesrc and
6864          * pfit state, we'll end up with a big fb scanned out into the wrong
6865          * sized surface.
6866          */
6867         intel_set_pipe_src_size(new_crtc_state);
6868
6869         /* on skylake this is done by detaching scalers */
6870         if (DISPLAY_VER(dev_priv) >= 9) {
6871                 if (new_crtc_state->pch_pfit.enabled)
6872                         skl_pfit_enable(new_crtc_state);
6873         } else if (HAS_PCH_SPLIT(dev_priv)) {
6874                 if (new_crtc_state->pch_pfit.enabled)
6875                         ilk_pfit_enable(new_crtc_state);
6876                 else if (old_crtc_state->pch_pfit.enabled)
6877                         ilk_pfit_disable(old_crtc_state);
6878         }
6879
6880         /*
6881          * The register is supposedly single buffered so perhaps
6882          * not 100% correct to do this here. But SKL+ calculate
6883          * this based on the adjust pixel rate so pfit changes do
6884          * affect it and so it must be updated for fastsets.
6885          * HSW/BDW only really need this here for fastboot, after
6886          * that the value should not change without a full modeset.
6887          */
6888         if (DISPLAY_VER(dev_priv) >= 9 ||
6889             IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
6890                 hsw_set_linetime_wm(new_crtc_state);
6891
6892         if (new_crtc_state->seamless_m_n)
6893                 intel_cpu_transcoder_set_m1_n1(crtc, new_crtc_state->cpu_transcoder,
6894                                                &new_crtc_state->dp_m_n);
6895 }
6896
6897 static void commit_pipe_pre_planes(struct intel_atomic_state *state,
6898                                    struct intel_crtc *crtc)
6899 {
6900         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6901         const struct intel_crtc_state *old_crtc_state =
6902                 intel_atomic_get_old_crtc_state(state, crtc);
6903         const struct intel_crtc_state *new_crtc_state =
6904                 intel_atomic_get_new_crtc_state(state, crtc);
6905         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
6906
6907         /*
6908          * During modesets pipe configuration was programmed as the
6909          * CRTC was enabled.
6910          */
6911         if (!modeset) {
6912                 if (intel_crtc_needs_color_update(new_crtc_state))
6913                         intel_color_commit_arm(new_crtc_state);
6914
6915                 if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
6916                         bdw_set_pipe_misc(new_crtc_state);
6917
6918                 if (intel_crtc_needs_fastset(new_crtc_state))
6919                         intel_pipe_fastset(old_crtc_state, new_crtc_state);
6920         }
6921
6922         intel_psr2_program_trans_man_trk_ctl(new_crtc_state);
6923
6924         intel_atomic_update_watermarks(state, crtc);
6925 }
6926
6927 static void commit_pipe_post_planes(struct intel_atomic_state *state,
6928                                     struct intel_crtc *crtc)
6929 {
6930         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6931         const struct intel_crtc_state *new_crtc_state =
6932                 intel_atomic_get_new_crtc_state(state, crtc);
6933
6934         /*
6935          * Disable the scaler(s) after the plane(s) so that we don't
6936          * get a catastrophic underrun even if the two operations
6937          * end up happening in two different frames.
6938          */
6939         if (DISPLAY_VER(dev_priv) >= 9 &&
6940             !intel_crtc_needs_modeset(new_crtc_state))
6941                 skl_detach_scalers(new_crtc_state);
6942 }
6943
6944 static void intel_enable_crtc(struct intel_atomic_state *state,
6945                               struct intel_crtc *crtc)
6946 {
6947         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6948         const struct intel_crtc_state *new_crtc_state =
6949                 intel_atomic_get_new_crtc_state(state, crtc);
6950
6951         if (!intel_crtc_needs_modeset(new_crtc_state))
6952                 return;
6953
6954         intel_crtc_update_active_timings(new_crtc_state);
6955
6956         dev_priv->display.funcs.display->crtc_enable(state, crtc);
6957
6958         if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
6959                 return;
6960
6961         /* vblanks work again, re-enable pipe CRC. */
6962         intel_crtc_enable_pipe_crc(crtc);
6963 }
6964
6965 static void intel_update_crtc(struct intel_atomic_state *state,
6966                               struct intel_crtc *crtc)
6967 {
6968         struct drm_i915_private *i915 = to_i915(state->base.dev);
6969         const struct intel_crtc_state *old_crtc_state =
6970                 intel_atomic_get_old_crtc_state(state, crtc);
6971         struct intel_crtc_state *new_crtc_state =
6972                 intel_atomic_get_new_crtc_state(state, crtc);
6973         bool modeset = intel_crtc_needs_modeset(new_crtc_state);
6974
6975         if (!modeset) {
6976                 if (new_crtc_state->preload_luts &&
6977                     intel_crtc_needs_color_update(new_crtc_state))
6978                         intel_color_load_luts(new_crtc_state);
6979
6980                 intel_pre_plane_update(state, crtc);
6981
6982                 if (intel_crtc_needs_fastset(new_crtc_state))
6983                         intel_encoders_update_pipe(state, crtc);
6984
6985                 if (DISPLAY_VER(i915) >= 11 &&
6986                     intel_crtc_needs_fastset(new_crtc_state))
6987                         icl_set_pipe_chicken(new_crtc_state);
6988         }
6989
6990         intel_fbc_update(state, crtc);
6991
6992         drm_WARN_ON(&i915->drm, !intel_display_power_is_enabled(i915, POWER_DOMAIN_DC_OFF));
6993
6994         if (!modeset &&
6995             intel_crtc_needs_color_update(new_crtc_state))
6996                 intel_color_commit_noarm(new_crtc_state);
6997
6998         intel_crtc_planes_update_noarm(state, crtc);
6999
7000         /* Perform vblank evasion around commit operation */
7001         intel_pipe_update_start(new_crtc_state);
7002
7003         commit_pipe_pre_planes(state, crtc);
7004
7005         intel_crtc_planes_update_arm(state, crtc);
7006
7007         commit_pipe_post_planes(state, crtc);
7008
7009         intel_pipe_update_end(new_crtc_state);
7010
7011         /*
7012          * We usually enable FIFO underrun interrupts as part of the
7013          * CRTC enable sequence during modesets.  But when we inherit a
7014          * valid pipe configuration from the BIOS we need to take care
7015          * of enabling them on the CRTC's first fastset.
7016          */
7017         if (intel_crtc_needs_fastset(new_crtc_state) && !modeset &&
7018             old_crtc_state->inherited)
7019                 intel_crtc_arm_fifo_underrun(crtc, new_crtc_state);
7020 }
7021
7022 static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
7023                                           struct intel_crtc_state *old_crtc_state,
7024                                           struct intel_crtc_state *new_crtc_state,
7025                                           struct intel_crtc *crtc)
7026 {
7027         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7028
7029         /*
7030          * We need to disable pipe CRC before disabling the pipe,
7031          * or we race against vblank off.
7032          */
7033         intel_crtc_disable_pipe_crc(crtc);
7034
7035         dev_priv->display.funcs.display->crtc_disable(state, crtc);
7036         crtc->active = false;
7037         intel_fbc_disable(crtc);
7038         intel_disable_shared_dpll(old_crtc_state);
7039
7040         if (!new_crtc_state->hw.active)
7041                 intel_initial_watermarks(state, crtc);
7042 }
7043
7044 static void intel_commit_modeset_disables(struct intel_atomic_state *state)
7045 {
7046         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
7047         struct intel_crtc *crtc;
7048         u32 handled = 0;
7049         int i;
7050
7051         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
7052                                             new_crtc_state, i) {
7053                 if (!intel_crtc_needs_modeset(new_crtc_state))
7054                         continue;
7055
7056                 if (!old_crtc_state->hw.active)
7057                         continue;
7058
7059                 intel_pre_plane_update(state, crtc);
7060                 intel_crtc_disable_planes(state, crtc);
7061         }
7062
7063         /* Only disable port sync and MST slaves */
7064         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
7065                                             new_crtc_state, i) {
7066                 if (!intel_crtc_needs_modeset(new_crtc_state))
7067                         continue;
7068
7069                 if (!old_crtc_state->hw.active)
7070                         continue;
7071
7072                 /* In case of Transcoder port Sync master slave CRTCs can be
7073                  * assigned in any order and we need to make sure that
7074                  * slave CRTCs are disabled first and then master CRTC since
7075                  * Slave vblanks are masked till Master Vblanks.
7076                  */
7077                 if (!is_trans_port_sync_slave(old_crtc_state) &&
7078                     !intel_dp_mst_is_slave_trans(old_crtc_state) &&
7079                     !intel_crtc_is_bigjoiner_slave(old_crtc_state))
7080                         continue;
7081
7082                 intel_old_crtc_state_disables(state, old_crtc_state,
7083                                               new_crtc_state, crtc);
7084                 handled |= BIT(crtc->pipe);
7085         }
7086
7087         /* Disable everything else left on */
7088         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
7089                                             new_crtc_state, i) {
7090                 if (!intel_crtc_needs_modeset(new_crtc_state) ||
7091                     (handled & BIT(crtc->pipe)))
7092                         continue;
7093
7094                 if (!old_crtc_state->hw.active)
7095                         continue;
7096
7097                 intel_old_crtc_state_disables(state, old_crtc_state,
7098                                               new_crtc_state, crtc);
7099         }
7100 }
7101
7102 static void intel_commit_modeset_enables(struct intel_atomic_state *state)
7103 {
7104         struct intel_crtc_state *new_crtc_state;
7105         struct intel_crtc *crtc;
7106         int i;
7107
7108         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
7109                 if (!new_crtc_state->hw.active)
7110                         continue;
7111
7112                 intel_enable_crtc(state, crtc);
7113                 intel_update_crtc(state, crtc);
7114         }
7115 }
7116
7117 static void skl_commit_modeset_enables(struct intel_atomic_state *state)
7118 {
7119         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
7120         struct intel_crtc *crtc;
7121         struct intel_crtc_state *old_crtc_state, *new_crtc_state;
7122         struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
7123         u8 update_pipes = 0, modeset_pipes = 0;
7124         int i;
7125
7126         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
7127                 enum pipe pipe = crtc->pipe;
7128
7129                 if (!new_crtc_state->hw.active)
7130                         continue;
7131
7132                 /* ignore allocations for crtc's that have been turned off. */
7133                 if (!intel_crtc_needs_modeset(new_crtc_state)) {
7134                         entries[pipe] = old_crtc_state->wm.skl.ddb;
7135                         update_pipes |= BIT(pipe);
7136                 } else {
7137                         modeset_pipes |= BIT(pipe);
7138                 }
7139         }
7140
7141         /*
7142          * Whenever the number of active pipes changes, we need to make sure we
7143          * update the pipes in the right order so that their ddb allocations
7144          * never overlap with each other between CRTC updates. Otherwise we'll
7145          * cause pipe underruns and other bad stuff.
7146          *
7147          * So first lets enable all pipes that do not need a fullmodeset as
7148          * those don't have any external dependency.
7149          */
7150         while (update_pipes) {
7151                 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
7152                                                     new_crtc_state, i) {
7153                         enum pipe pipe = crtc->pipe;
7154
7155                         if ((update_pipes & BIT(pipe)) == 0)
7156                                 continue;
7157
7158                         if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
7159                                                         entries, I915_MAX_PIPES, pipe))
7160                                 continue;
7161
7162                         entries[pipe] = new_crtc_state->wm.skl.ddb;
7163                         update_pipes &= ~BIT(pipe);
7164
7165                         intel_update_crtc(state, crtc);
7166
7167                         /*
7168                          * If this is an already active pipe, it's DDB changed,
7169                          * and this isn't the last pipe that needs updating
7170                          * then we need to wait for a vblank to pass for the
7171                          * new ddb allocation to take effect.
7172                          */
7173                         if (!skl_ddb_entry_equal(&new_crtc_state->wm.skl.ddb,
7174                                                  &old_crtc_state->wm.skl.ddb) &&
7175                             (update_pipes | modeset_pipes))
7176                                 intel_crtc_wait_for_next_vblank(crtc);
7177                 }
7178         }
7179
7180         update_pipes = modeset_pipes;
7181
7182         /*
7183          * Enable all pipes that needs a modeset and do not depends on other
7184          * pipes
7185          */
7186         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
7187                 enum pipe pipe = crtc->pipe;
7188
7189                 if ((modeset_pipes & BIT(pipe)) == 0)
7190                         continue;
7191
7192                 if (intel_dp_mst_is_slave_trans(new_crtc_state) ||
7193                     is_trans_port_sync_master(new_crtc_state) ||
7194                     intel_crtc_is_bigjoiner_master(new_crtc_state))
7195                         continue;
7196
7197                 modeset_pipes &= ~BIT(pipe);
7198
7199                 intel_enable_crtc(state, crtc);
7200         }
7201
7202         /*
7203          * Then we enable all remaining pipes that depend on other
7204          * pipes: MST slaves and port sync masters, big joiner master
7205          */
7206         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
7207                 enum pipe pipe = crtc->pipe;
7208
7209                 if ((modeset_pipes & BIT(pipe)) == 0)
7210                         continue;
7211
7212                 modeset_pipes &= ~BIT(pipe);
7213
7214                 intel_enable_crtc(state, crtc);
7215         }
7216
7217         /*
7218          * Finally we do the plane updates/etc. for all pipes that got enabled.
7219          */
7220         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
7221                 enum pipe pipe = crtc->pipe;
7222
7223                 if ((update_pipes & BIT(pipe)) == 0)
7224                         continue;
7225
7226                 drm_WARN_ON(&dev_priv->drm, skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb,
7227                                                                         entries, I915_MAX_PIPES, pipe));
7228
7229                 entries[pipe] = new_crtc_state->wm.skl.ddb;
7230                 update_pipes &= ~BIT(pipe);
7231
7232                 intel_update_crtc(state, crtc);
7233         }
7234
7235         drm_WARN_ON(&dev_priv->drm, modeset_pipes);
7236         drm_WARN_ON(&dev_priv->drm, update_pipes);
7237 }
7238
7239 static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
7240 {
7241         struct intel_atomic_state *state, *next;
7242         struct llist_node *freed;
7243
7244         freed = llist_del_all(&dev_priv->display.atomic_helper.free_list);
7245         llist_for_each_entry_safe(state, next, freed, freed)
7246                 drm_atomic_state_put(&state->base);
7247 }
7248
7249 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
7250 {
7251         struct drm_i915_private *dev_priv =
7252                 container_of(work, typeof(*dev_priv), display.atomic_helper.free_work);
7253
7254         intel_atomic_helper_free_state(dev_priv);
7255 }
7256
7257 static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_state)
7258 {
7259         struct wait_queue_entry wait_fence, wait_reset;
7260         struct drm_i915_private *dev_priv = to_i915(intel_state->base.dev);
7261
7262         init_wait_entry(&wait_fence, 0);
7263         init_wait_entry(&wait_reset, 0);
7264         for (;;) {
7265                 prepare_to_wait(&intel_state->commit_ready.wait,
7266                                 &wait_fence, TASK_UNINTERRUPTIBLE);
7267                 prepare_to_wait(bit_waitqueue(&to_gt(dev_priv)->reset.flags,
7268                                               I915_RESET_MODESET),
7269                                 &wait_reset, TASK_UNINTERRUPTIBLE);
7270
7271
7272                 if (i915_sw_fence_done(&intel_state->commit_ready) ||
7273                     test_bit(I915_RESET_MODESET, &to_gt(dev_priv)->reset.flags))
7274                         break;
7275
7276                 schedule();
7277         }
7278         finish_wait(&intel_state->commit_ready.wait, &wait_fence);
7279         finish_wait(bit_waitqueue(&to_gt(dev_priv)->reset.flags,
7280                                   I915_RESET_MODESET),
7281                     &wait_reset);
7282 }
7283
7284 static void intel_atomic_cleanup_work(struct work_struct *work)
7285 {
7286         struct intel_atomic_state *state =
7287                 container_of(work, struct intel_atomic_state, base.commit_work);
7288         struct drm_i915_private *i915 = to_i915(state->base.dev);
7289         struct intel_crtc_state *old_crtc_state;
7290         struct intel_crtc *crtc;
7291         int i;
7292
7293         for_each_old_intel_crtc_in_state(state, crtc, old_crtc_state, i)
7294                 intel_color_cleanup_commit(old_crtc_state);
7295
7296         drm_atomic_helper_cleanup_planes(&i915->drm, &state->base);
7297         drm_atomic_helper_commit_cleanup_done(&state->base);
7298         drm_atomic_state_put(&state->base);
7299
7300         intel_atomic_helper_free_state(i915);
7301 }
7302
7303 static void intel_atomic_prepare_plane_clear_colors(struct intel_atomic_state *state)
7304 {
7305         struct drm_i915_private *i915 = to_i915(state->base.dev);
7306         struct intel_plane *plane;
7307         struct intel_plane_state *plane_state;
7308         int i;
7309
7310         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
7311                 struct drm_framebuffer *fb = plane_state->hw.fb;
7312                 int cc_plane;
7313                 int ret;
7314
7315                 if (!fb)
7316                         continue;
7317
7318                 cc_plane = intel_fb_rc_ccs_cc_plane(fb);
7319                 if (cc_plane < 0)
7320                         continue;
7321
7322                 /*
7323                  * The layout of the fast clear color value expected by HW
7324                  * (the DRM ABI requiring this value to be located in fb at
7325                  * offset 0 of cc plane, plane #2 previous generations or
7326                  * plane #1 for flat ccs):
7327                  * - 4 x 4 bytes per-channel value
7328                  *   (in surface type specific float/int format provided by the fb user)
7329                  * - 8 bytes native color value used by the display
7330                  *   (converted/written by GPU during a fast clear operation using the
7331                  *    above per-channel values)
7332                  *
7333                  * The commit's FB prepare hook already ensured that FB obj is pinned and the
7334                  * caller made sure that the object is synced wrt. the related color clear value
7335                  * GPU write on it.
7336                  */
7337                 ret = i915_gem_object_read_from_page(intel_fb_obj(fb),
7338                                                      fb->offsets[cc_plane] + 16,
7339                                                      &plane_state->ccval,
7340                                                      sizeof(plane_state->ccval));
7341                 /* The above could only fail if the FB obj has an unexpected backing store type. */
7342                 drm_WARN_ON(&i915->drm, ret);
7343         }
7344 }
7345
7346 static void intel_atomic_commit_tail(struct intel_atomic_state *state)
7347 {
7348         struct drm_device *dev = state->base.dev;
7349         struct drm_i915_private *dev_priv = to_i915(dev);
7350         struct intel_crtc_state *new_crtc_state, *old_crtc_state;
7351         struct intel_crtc *crtc;
7352         struct intel_power_domain_mask put_domains[I915_MAX_PIPES] = {};
7353         intel_wakeref_t wakeref = 0;
7354         int i;
7355
7356         intel_atomic_commit_fence_wait(state);
7357
7358         drm_atomic_helper_wait_for_dependencies(&state->base);
7359         drm_dp_mst_atomic_wait_for_dependencies(&state->base);
7360
7361         /*
7362          * During full modesets we write a lot of registers, wait
7363          * for PLLs, etc. Doing that while DC states are enabled
7364          * is not a good idea.
7365          *
7366          * During fastsets and other updates we also need to
7367          * disable DC states due to the following scenario:
7368          * 1. DC5 exit and PSR exit happen
7369          * 2. Some or all _noarm() registers are written
7370          * 3. Due to some long delay PSR is re-entered
7371          * 4. DC5 entry -> DMC saves the already written new
7372          *    _noarm() registers and the old not yet written
7373          *    _arm() registers
7374          * 5. DC5 exit -> DMC restores a mixture of old and
7375          *    new register values and arms the update
7376          * 6. PSR exit -> hardware latches a mixture of old and
7377          *    new register values -> corrupted frame, or worse
7378          * 7. New _arm() registers are finally written
7379          * 8. Hardware finally latches a complete set of new
7380          *    register values, and subsequent frames will be OK again
7381          */
7382         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DC_OFF);
7383
7384         intel_atomic_prepare_plane_clear_colors(state);
7385
7386         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
7387                                             new_crtc_state, i) {
7388                 if (intel_crtc_needs_modeset(new_crtc_state) ||
7389                     intel_crtc_needs_fastset(new_crtc_state))
7390                         intel_modeset_get_crtc_power_domains(new_crtc_state, &put_domains[crtc->pipe]);
7391         }
7392
7393         intel_commit_modeset_disables(state);
7394
7395         /* FIXME: Eventually get rid of our crtc->config pointer */
7396         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
7397                 crtc->config = new_crtc_state;
7398
7399         if (state->modeset) {
7400                 drm_atomic_helper_update_legacy_modeset_state(dev, &state->base);
7401
7402                 intel_set_cdclk_pre_plane_update(state);
7403
7404                 intel_modeset_verify_disabled(dev_priv, state);
7405         }
7406
7407         intel_sagv_pre_plane_update(state);
7408
7409         /* Complete the events for pipes that have now been disabled */
7410         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
7411                 bool modeset = intel_crtc_needs_modeset(new_crtc_state);
7412
7413                 /* Complete events for now disable pipes here. */
7414                 if (modeset && !new_crtc_state->hw.active && new_crtc_state->uapi.event) {
7415                         spin_lock_irq(&dev->event_lock);
7416                         drm_crtc_send_vblank_event(&crtc->base,
7417                                                    new_crtc_state->uapi.event);
7418                         spin_unlock_irq(&dev->event_lock);
7419
7420                         new_crtc_state->uapi.event = NULL;
7421                 }
7422         }
7423
7424         intel_encoders_update_prepare(state);
7425
7426         intel_dbuf_pre_plane_update(state);
7427         intel_mbus_dbox_update(state);
7428
7429         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
7430                 if (new_crtc_state->do_async_flip)
7431                         intel_crtc_enable_flip_done(state, crtc);
7432         }
7433
7434         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
7435         dev_priv->display.funcs.display->commit_modeset_enables(state);
7436
7437         intel_encoders_update_complete(state);
7438
7439         if (state->modeset)
7440                 intel_set_cdclk_post_plane_update(state);
7441
7442         intel_wait_for_vblank_workers(state);
7443
7444         /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
7445          * already, but still need the state for the delayed optimization. To
7446          * fix this:
7447          * - wrap the optimization/post_plane_update stuff into a per-crtc work.
7448          * - schedule that vblank worker _before_ calling hw_done
7449          * - at the start of commit_tail, cancel it _synchrously
7450          * - switch over to the vblank wait helper in the core after that since
7451          *   we don't need out special handling any more.
7452          */
7453         drm_atomic_helper_wait_for_flip_done(dev, &state->base);
7454
7455         for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
7456                 if (new_crtc_state->do_async_flip)
7457                         intel_crtc_disable_flip_done(state, crtc);
7458         }
7459
7460         /*
7461          * Now that the vblank has passed, we can go ahead and program the
7462          * optimal watermarks on platforms that need two-step watermark
7463          * programming.
7464          *
7465          * TODO: Move this (and other cleanup) to an async worker eventually.
7466          */
7467         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
7468                                             new_crtc_state, i) {
7469                 /*
7470                  * Gen2 reports pipe underruns whenever all planes are disabled.
7471                  * So re-enable underrun reporting after some planes get enabled.
7472                  *
7473                  * We do this before .optimize_watermarks() so that we have a
7474                  * chance of catching underruns with the intermediate watermarks
7475                  * vs. the new plane configuration.
7476                  */
7477                 if (DISPLAY_VER(dev_priv) == 2 && planes_enabling(old_crtc_state, new_crtc_state))
7478                         intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true);
7479
7480                 intel_optimize_watermarks(state, crtc);
7481         }
7482
7483         intel_dbuf_post_plane_update(state);
7484         intel_psr_post_plane_update(state);
7485
7486         for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
7487                 intel_post_plane_update(state, crtc);
7488
7489                 intel_modeset_put_crtc_power_domains(crtc, &put_domains[crtc->pipe]);
7490
7491                 intel_modeset_verify_crtc(crtc, state, old_crtc_state, new_crtc_state);
7492
7493                 /* Must be done after gamma readout due to HSW split gamma vs. IPS w/a */
7494                 hsw_ips_post_update(state, crtc);
7495
7496                 /*
7497                  * Activate DRRS after state readout to avoid
7498                  * dp_m_n vs. dp_m2_n2 confusion on BDW+.
7499                  */
7500                 intel_drrs_activate(new_crtc_state);
7501
7502                 /*
7503                  * DSB cleanup is done in cleanup_work aligning with framebuffer
7504                  * cleanup. So copy and reset the dsb structure to sync with
7505                  * commit_done and later do dsb cleanup in cleanup_work.
7506                  *
7507                  * FIXME get rid of this funny new->old swapping
7508                  */
7509                 old_crtc_state->dsb = fetch_and_zero(&new_crtc_state->dsb);
7510         }
7511
7512         /* Underruns don't always raise interrupts, so check manually */
7513         intel_check_cpu_fifo_underruns(dev_priv);
7514         intel_check_pch_fifo_underruns(dev_priv);
7515
7516         if (state->modeset)
7517                 intel_verify_planes(state);
7518
7519         intel_sagv_post_plane_update(state);
7520
7521         drm_atomic_helper_commit_hw_done(&state->base);
7522
7523         if (state->modeset) {
7524                 /* As one of the primary mmio accessors, KMS has a high
7525                  * likelihood of triggering bugs in unclaimed access. After we
7526                  * finish modesetting, see if an error has been flagged, and if
7527                  * so enable debugging for the next modeset - and hope we catch
7528                  * the culprit.
7529                  */
7530                 intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore);
7531         }
7532         intel_display_power_put(dev_priv, POWER_DOMAIN_DC_OFF, wakeref);
7533         intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
7534
7535         /*
7536          * Defer the cleanup of the old state to a separate worker to not
7537          * impede the current task (userspace for blocking modesets) that
7538          * are executed inline. For out-of-line asynchronous modesets/flips,
7539          * deferring to a new worker seems overkill, but we would place a
7540          * schedule point (cond_resched()) here anyway to keep latencies
7541          * down.
7542          */
7543         INIT_WORK(&state->base.commit_work, intel_atomic_cleanup_work);
7544         queue_work(system_highpri_wq, &state->base.commit_work);
7545 }
7546
7547 static void intel_atomic_commit_work(struct work_struct *work)
7548 {
7549         struct intel_atomic_state *state =
7550                 container_of(work, struct intel_atomic_state, base.commit_work);
7551
7552         intel_atomic_commit_tail(state);
7553 }
7554
7555 static int
7556 intel_atomic_commit_ready(struct i915_sw_fence *fence,
7557                           enum i915_sw_fence_notify notify)
7558 {
7559         struct intel_atomic_state *state =
7560                 container_of(fence, struct intel_atomic_state, commit_ready);
7561
7562         switch (notify) {
7563         case FENCE_COMPLETE:
7564                 /* we do blocking waits in the worker, nothing to do here */
7565                 break;
7566         case FENCE_FREE:
7567                 {
7568                         struct intel_atomic_helper *helper =
7569                                 &to_i915(state->base.dev)->display.atomic_helper;
7570
7571                         if (llist_add(&state->freed, &helper->free_list))
7572                                 schedule_work(&helper->free_work);
7573                         break;
7574                 }
7575         }
7576
7577         return NOTIFY_DONE;
7578 }
7579
7580 static void intel_atomic_track_fbs(struct intel_atomic_state *state)
7581 {
7582         struct intel_plane_state *old_plane_state, *new_plane_state;
7583         struct intel_plane *plane;
7584         int i;
7585
7586         for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
7587                                              new_plane_state, i)
7588                 intel_frontbuffer_track(to_intel_frontbuffer(old_plane_state->hw.fb),
7589                                         to_intel_frontbuffer(new_plane_state->hw.fb),
7590                                         plane->frontbuffer_bit);
7591 }
7592
7593 static int intel_atomic_commit(struct drm_device *dev,
7594                                struct drm_atomic_state *_state,
7595                                bool nonblock)
7596 {
7597         struct intel_atomic_state *state = to_intel_atomic_state(_state);
7598         struct drm_i915_private *dev_priv = to_i915(dev);
7599         int ret = 0;
7600
7601         state->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
7602
7603         drm_atomic_state_get(&state->base);
7604         i915_sw_fence_init(&state->commit_ready,
7605                            intel_atomic_commit_ready);
7606
7607         /*
7608          * The intel_legacy_cursor_update() fast path takes care
7609          * of avoiding the vblank waits for simple cursor
7610          * movement and flips. For cursor on/off and size changes,
7611          * we want to perform the vblank waits so that watermark
7612          * updates happen during the correct frames. Gen9+ have
7613          * double buffered watermarks and so shouldn't need this.
7614          *
7615          * Unset state->legacy_cursor_update before the call to
7616          * drm_atomic_helper_setup_commit() because otherwise
7617          * drm_atomic_helper_wait_for_flip_done() is a noop and
7618          * we get FIFO underruns because we didn't wait
7619          * for vblank.
7620          *
7621          * FIXME doing watermarks and fb cleanup from a vblank worker
7622          * (assuming we had any) would solve these problems.
7623          */
7624         if (DISPLAY_VER(dev_priv) < 9 && state->base.legacy_cursor_update) {
7625                 struct intel_crtc_state *new_crtc_state;
7626                 struct intel_crtc *crtc;
7627                 int i;
7628
7629                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
7630                         if (new_crtc_state->wm.need_postvbl_update ||
7631                             new_crtc_state->update_wm_post)
7632                                 state->base.legacy_cursor_update = false;
7633         }
7634
7635         ret = intel_atomic_prepare_commit(state);
7636         if (ret) {
7637                 drm_dbg_atomic(&dev_priv->drm,
7638                                "Preparing state failed with %i\n", ret);
7639                 i915_sw_fence_commit(&state->commit_ready);
7640                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
7641                 return ret;
7642         }
7643
7644         ret = drm_atomic_helper_setup_commit(&state->base, nonblock);
7645         if (!ret)
7646                 ret = drm_atomic_helper_swap_state(&state->base, true);
7647         if (!ret)
7648                 intel_atomic_swap_global_state(state);
7649
7650         if (ret) {
7651                 struct intel_crtc_state *new_crtc_state;
7652                 struct intel_crtc *crtc;
7653                 int i;
7654
7655                 i915_sw_fence_commit(&state->commit_ready);
7656
7657                 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i)
7658                         intel_color_cleanup_commit(new_crtc_state);
7659
7660                 drm_atomic_helper_cleanup_planes(dev, &state->base);
7661                 intel_runtime_pm_put(&dev_priv->runtime_pm, state->wakeref);
7662                 return ret;
7663         }
7664         intel_shared_dpll_swap_state(state);
7665         intel_atomic_track_fbs(state);
7666
7667         drm_atomic_state_get(&state->base);
7668         INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
7669
7670         i915_sw_fence_commit(&state->commit_ready);
7671         if (nonblock && state->modeset) {
7672                 queue_work(dev_priv->display.wq.modeset, &state->base.commit_work);
7673         } else if (nonblock) {
7674                 queue_work(dev_priv->display.wq.flip, &state->base.commit_work);
7675         } else {
7676                 if (state->modeset)
7677                         flush_workqueue(dev_priv->display.wq.modeset);
7678                 intel_atomic_commit_tail(state);
7679         }
7680
7681         return 0;
7682 }
7683
7684 /**
7685  * intel_plane_destroy - destroy a plane
7686  * @plane: plane to destroy
7687  *
7688  * Common destruction function for all types of planes (primary, cursor,
7689  * sprite).
7690  */
7691 void intel_plane_destroy(struct drm_plane *plane)
7692 {
7693         drm_plane_cleanup(plane);
7694         kfree(to_intel_plane(plane));
7695 }
7696
7697 static void intel_plane_possible_crtcs_init(struct drm_i915_private *dev_priv)
7698 {
7699         struct intel_plane *plane;
7700
7701         for_each_intel_plane(&dev_priv->drm, plane) {
7702                 struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv,
7703                                                               plane->pipe);
7704
7705                 plane->base.possible_crtcs = drm_crtc_mask(&crtc->base);
7706         }
7707 }
7708
7709
7710 int intel_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
7711                                       struct drm_file *file)
7712 {
7713         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
7714         struct drm_crtc *drmmode_crtc;
7715         struct intel_crtc *crtc;
7716
7717         drmmode_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
7718         if (!drmmode_crtc)
7719                 return -ENOENT;
7720
7721         crtc = to_intel_crtc(drmmode_crtc);
7722         pipe_from_crtc_id->pipe = crtc->pipe;
7723
7724         return 0;
7725 }
7726
7727 static u32 intel_encoder_possible_clones(struct intel_encoder *encoder)
7728 {
7729         struct drm_device *dev = encoder->base.dev;
7730         struct intel_encoder *source_encoder;
7731         u32 possible_clones = 0;
7732
7733         for_each_intel_encoder(dev, source_encoder) {
7734                 if (encoders_cloneable(encoder, source_encoder))
7735                         possible_clones |= drm_encoder_mask(&source_encoder->base);
7736         }
7737
7738         return possible_clones;
7739 }
7740
7741 static u32 intel_encoder_possible_crtcs(struct intel_encoder *encoder)
7742 {
7743         struct drm_device *dev = encoder->base.dev;
7744         struct intel_crtc *crtc;
7745         u32 possible_crtcs = 0;
7746
7747         for_each_intel_crtc_in_pipe_mask(dev, crtc, encoder->pipe_mask)
7748                 possible_crtcs |= drm_crtc_mask(&crtc->base);
7749
7750         return possible_crtcs;
7751 }
7752
7753 static bool ilk_has_edp_a(struct drm_i915_private *dev_priv)
7754 {
7755         if (!IS_MOBILE(dev_priv))
7756                 return false;
7757
7758         if ((intel_de_read(dev_priv, DP_A) & DP_DETECTED) == 0)
7759                 return false;
7760
7761         if (IS_IRONLAKE(dev_priv) && (intel_de_read(dev_priv, FUSE_STRAP) & ILK_eDP_A_DISABLE))
7762                 return false;
7763
7764         return true;
7765 }
7766
7767 static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
7768 {
7769         if (DISPLAY_VER(dev_priv) >= 9)
7770                 return false;
7771
7772         if (IS_HSW_ULT(dev_priv) || IS_BDW_ULT(dev_priv))
7773                 return false;
7774
7775         if (HAS_PCH_LPT_H(dev_priv) &&
7776             intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
7777                 return false;
7778
7779         /* DDI E can't be used if DDI A requires 4 lanes */
7780         if (intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
7781                 return false;
7782
7783         if (!dev_priv->display.vbt.int_crt_support)
7784                 return false;
7785
7786         return true;
7787 }
7788
7789 static void intel_setup_outputs(struct drm_i915_private *dev_priv)
7790 {
7791         struct intel_encoder *encoder;
7792         bool dpd_is_edp = false;
7793
7794         intel_pps_unlock_regs_wa(dev_priv);
7795
7796         if (!HAS_DISPLAY(dev_priv))
7797                 return;
7798
7799         if (IS_DG2(dev_priv)) {
7800                 intel_ddi_init(dev_priv, PORT_A);
7801                 intel_ddi_init(dev_priv, PORT_B);
7802                 intel_ddi_init(dev_priv, PORT_C);
7803                 intel_ddi_init(dev_priv, PORT_D_XELPD);
7804                 intel_ddi_init(dev_priv, PORT_TC1);
7805         } else if (IS_ALDERLAKE_P(dev_priv)) {
7806                 intel_ddi_init(dev_priv, PORT_A);
7807                 intel_ddi_init(dev_priv, PORT_B);
7808                 intel_ddi_init(dev_priv, PORT_TC1);
7809                 intel_ddi_init(dev_priv, PORT_TC2);
7810                 intel_ddi_init(dev_priv, PORT_TC3);
7811                 intel_ddi_init(dev_priv, PORT_TC4);
7812                 icl_dsi_init(dev_priv);
7813         } else if (IS_ALDERLAKE_S(dev_priv)) {
7814                 intel_ddi_init(dev_priv, PORT_A);
7815                 intel_ddi_init(dev_priv, PORT_TC1);
7816                 intel_ddi_init(dev_priv, PORT_TC2);
7817                 intel_ddi_init(dev_priv, PORT_TC3);
7818                 intel_ddi_init(dev_priv, PORT_TC4);
7819         } else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv)) {
7820                 intel_ddi_init(dev_priv, PORT_A);
7821                 intel_ddi_init(dev_priv, PORT_B);
7822                 intel_ddi_init(dev_priv, PORT_TC1);
7823                 intel_ddi_init(dev_priv, PORT_TC2);
7824         } else if (DISPLAY_VER(dev_priv) >= 12) {
7825                 intel_ddi_init(dev_priv, PORT_A);
7826                 intel_ddi_init(dev_priv, PORT_B);
7827                 intel_ddi_init(dev_priv, PORT_TC1);
7828                 intel_ddi_init(dev_priv, PORT_TC2);
7829                 intel_ddi_init(dev_priv, PORT_TC3);
7830                 intel_ddi_init(dev_priv, PORT_TC4);
7831                 intel_ddi_init(dev_priv, PORT_TC5);
7832                 intel_ddi_init(dev_priv, PORT_TC6);
7833                 icl_dsi_init(dev_priv);
7834         } else if (IS_JSL_EHL(dev_priv)) {
7835                 intel_ddi_init(dev_priv, PORT_A);
7836                 intel_ddi_init(dev_priv, PORT_B);
7837                 intel_ddi_init(dev_priv, PORT_C);
7838                 intel_ddi_init(dev_priv, PORT_D);
7839                 icl_dsi_init(dev_priv);
7840         } else if (DISPLAY_VER(dev_priv) == 11) {
7841                 intel_ddi_init(dev_priv, PORT_A);
7842                 intel_ddi_init(dev_priv, PORT_B);
7843                 intel_ddi_init(dev_priv, PORT_C);
7844                 intel_ddi_init(dev_priv, PORT_D);
7845                 intel_ddi_init(dev_priv, PORT_E);
7846                 intel_ddi_init(dev_priv, PORT_F);
7847                 icl_dsi_init(dev_priv);
7848         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
7849                 intel_ddi_init(dev_priv, PORT_A);
7850                 intel_ddi_init(dev_priv, PORT_B);
7851                 intel_ddi_init(dev_priv, PORT_C);
7852                 vlv_dsi_init(dev_priv);
7853         } else if (DISPLAY_VER(dev_priv) >= 9) {
7854                 intel_ddi_init(dev_priv, PORT_A);
7855                 intel_ddi_init(dev_priv, PORT_B);
7856                 intel_ddi_init(dev_priv, PORT_C);
7857                 intel_ddi_init(dev_priv, PORT_D);
7858                 intel_ddi_init(dev_priv, PORT_E);
7859         } else if (HAS_DDI(dev_priv)) {
7860                 u32 found;
7861
7862                 if (intel_ddi_crt_present(dev_priv))
7863                         intel_crt_init(dev_priv);
7864
7865                 /* Haswell uses DDI functions to detect digital outputs. */
7866                 found = intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_INIT_DISPLAY_DETECTED;
7867                 if (found)
7868                         intel_ddi_init(dev_priv, PORT_A);
7869
7870                 found = intel_de_read(dev_priv, SFUSE_STRAP);
7871                 if (found & SFUSE_STRAP_DDIB_DETECTED)
7872                         intel_ddi_init(dev_priv, PORT_B);
7873                 if (found & SFUSE_STRAP_DDIC_DETECTED)
7874                         intel_ddi_init(dev_priv, PORT_C);
7875                 if (found & SFUSE_STRAP_DDID_DETECTED)
7876                         intel_ddi_init(dev_priv, PORT_D);
7877                 if (found & SFUSE_STRAP_DDIF_DETECTED)
7878                         intel_ddi_init(dev_priv, PORT_F);
7879         } else if (HAS_PCH_SPLIT(dev_priv)) {
7880                 int found;
7881
7882                 /*
7883                  * intel_edp_init_connector() depends on this completing first,
7884                  * to prevent the registration of both eDP and LVDS and the
7885                  * incorrect sharing of the PPS.
7886                  */
7887                 intel_lvds_init(dev_priv);
7888                 intel_crt_init(dev_priv);
7889
7890                 dpd_is_edp = intel_dp_is_port_edp(dev_priv, PORT_D);
7891
7892                 if (ilk_has_edp_a(dev_priv))
7893                         g4x_dp_init(dev_priv, DP_A, PORT_A);
7894
7895                 if (intel_de_read(dev_priv, PCH_HDMIB) & SDVO_DETECTED) {
7896                         /* PCH SDVOB multiplex with HDMIB */
7897                         found = intel_sdvo_init(dev_priv, PCH_SDVOB, PORT_B);
7898                         if (!found)
7899                                 g4x_hdmi_init(dev_priv, PCH_HDMIB, PORT_B);
7900                         if (!found && (intel_de_read(dev_priv, PCH_DP_B) & DP_DETECTED))
7901                                 g4x_dp_init(dev_priv, PCH_DP_B, PORT_B);
7902                 }
7903
7904                 if (intel_de_read(dev_priv, PCH_HDMIC) & SDVO_DETECTED)
7905                         g4x_hdmi_init(dev_priv, PCH_HDMIC, PORT_C);
7906
7907                 if (!dpd_is_edp && intel_de_read(dev_priv, PCH_HDMID) & SDVO_DETECTED)
7908                         g4x_hdmi_init(dev_priv, PCH_HDMID, PORT_D);
7909
7910                 if (intel_de_read(dev_priv, PCH_DP_C) & DP_DETECTED)
7911                         g4x_dp_init(dev_priv, PCH_DP_C, PORT_C);
7912
7913                 if (intel_de_read(dev_priv, PCH_DP_D) & DP_DETECTED)
7914                         g4x_dp_init(dev_priv, PCH_DP_D, PORT_D);
7915         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
7916                 bool has_edp, has_port;
7917
7918                 if (IS_VALLEYVIEW(dev_priv) && dev_priv->display.vbt.int_crt_support)
7919                         intel_crt_init(dev_priv);
7920
7921                 /*
7922                  * The DP_DETECTED bit is the latched state of the DDC
7923                  * SDA pin at boot. However since eDP doesn't require DDC
7924                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
7925                  * eDP ports may have been muxed to an alternate function.
7926                  * Thus we can't rely on the DP_DETECTED bit alone to detect
7927                  * eDP ports. Consult the VBT as well as DP_DETECTED to
7928                  * detect eDP ports.
7929                  *
7930                  * Sadly the straps seem to be missing sometimes even for HDMI
7931                  * ports (eg. on Voyo V3 - CHT x7-Z8700), so check both strap
7932                  * and VBT for the presence of the port. Additionally we can't
7933                  * trust the port type the VBT declares as we've seen at least
7934                  * HDMI ports that the VBT claim are DP or eDP.
7935                  */
7936                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_B);
7937                 has_port = intel_bios_is_port_present(dev_priv, PORT_B);
7938                 if (intel_de_read(dev_priv, VLV_DP_B) & DP_DETECTED || has_port)
7939                         has_edp &= g4x_dp_init(dev_priv, VLV_DP_B, PORT_B);
7940                 if ((intel_de_read(dev_priv, VLV_HDMIB) & SDVO_DETECTED || has_port) && !has_edp)
7941                         g4x_hdmi_init(dev_priv, VLV_HDMIB, PORT_B);
7942
7943                 has_edp = intel_dp_is_port_edp(dev_priv, PORT_C);
7944                 has_port = intel_bios_is_port_present(dev_priv, PORT_C);
7945                 if (intel_de_read(dev_priv, VLV_DP_C) & DP_DETECTED || has_port)
7946                         has_edp &= g4x_dp_init(dev_priv, VLV_DP_C, PORT_C);
7947                 if ((intel_de_read(dev_priv, VLV_HDMIC) & SDVO_DETECTED || has_port) && !has_edp)
7948                         g4x_hdmi_init(dev_priv, VLV_HDMIC, PORT_C);
7949
7950                 if (IS_CHERRYVIEW(dev_priv)) {
7951                         /*
7952                          * eDP not supported on port D,
7953                          * so no need to worry about it
7954                          */
7955                         has_port = intel_bios_is_port_present(dev_priv, PORT_D);
7956                         if (intel_de_read(dev_priv, CHV_DP_D) & DP_DETECTED || has_port)
7957                                 g4x_dp_init(dev_priv, CHV_DP_D, PORT_D);
7958                         if (intel_de_read(dev_priv, CHV_HDMID) & SDVO_DETECTED || has_port)
7959                                 g4x_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
7960                 }
7961
7962                 vlv_dsi_init(dev_priv);
7963         } else if (IS_PINEVIEW(dev_priv)) {
7964                 intel_lvds_init(dev_priv);
7965                 intel_crt_init(dev_priv);
7966         } else if (IS_DISPLAY_VER(dev_priv, 3, 4)) {
7967                 bool found = false;
7968
7969                 if (IS_MOBILE(dev_priv))
7970                         intel_lvds_init(dev_priv);
7971
7972                 intel_crt_init(dev_priv);
7973
7974                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
7975                         drm_dbg_kms(&dev_priv->drm, "probing SDVOB\n");
7976                         found = intel_sdvo_init(dev_priv, GEN3_SDVOB, PORT_B);
7977                         if (!found && IS_G4X(dev_priv)) {
7978                                 drm_dbg_kms(&dev_priv->drm,
7979                                             "probing HDMI on SDVOB\n");
7980                                 g4x_hdmi_init(dev_priv, GEN4_HDMIB, PORT_B);
7981                         }
7982
7983                         if (!found && IS_G4X(dev_priv))
7984                                 g4x_dp_init(dev_priv, DP_B, PORT_B);
7985                 }
7986
7987                 /* Before G4X SDVOC doesn't have its own detect register */
7988
7989                 if (intel_de_read(dev_priv, GEN3_SDVOB) & SDVO_DETECTED) {
7990                         drm_dbg_kms(&dev_priv->drm, "probing SDVOC\n");
7991                         found = intel_sdvo_init(dev_priv, GEN3_SDVOC, PORT_C);
7992                 }
7993
7994                 if (!found && (intel_de_read(dev_priv, GEN3_SDVOC) & SDVO_DETECTED)) {
7995
7996                         if (IS_G4X(dev_priv)) {
7997                                 drm_dbg_kms(&dev_priv->drm,
7998                                             "probing HDMI on SDVOC\n");
7999                                 g4x_hdmi_init(dev_priv, GEN4_HDMIC, PORT_C);
8000                         }
8001                         if (IS_G4X(dev_priv))
8002                                 g4x_dp_init(dev_priv, DP_C, PORT_C);
8003                 }
8004
8005                 if (IS_G4X(dev_priv) && (intel_de_read(dev_priv, DP_D) & DP_DETECTED))
8006                         g4x_dp_init(dev_priv, DP_D, PORT_D);
8007
8008                 if (SUPPORTS_TV(dev_priv))
8009                         intel_tv_init(dev_priv);
8010         } else if (DISPLAY_VER(dev_priv) == 2) {
8011                 if (IS_I85X(dev_priv))
8012                         intel_lvds_init(dev_priv);
8013
8014                 intel_crt_init(dev_priv);
8015                 intel_dvo_init(dev_priv);
8016         }
8017
8018         for_each_intel_encoder(&dev_priv->drm, encoder) {
8019                 encoder->base.possible_crtcs =
8020                         intel_encoder_possible_crtcs(encoder);
8021                 encoder->base.possible_clones =
8022                         intel_encoder_possible_clones(encoder);
8023         }
8024
8025         intel_init_pch_refclk(dev_priv);
8026
8027         drm_helper_move_panel_connectors_to_head(&dev_priv->drm);
8028 }
8029
8030 static int max_dotclock(struct drm_i915_private *i915)
8031 {
8032         int max_dotclock = i915->max_dotclk_freq;
8033
8034         /* icl+ might use bigjoiner */
8035         if (DISPLAY_VER(i915) >= 11)
8036                 max_dotclock *= 2;
8037
8038         return max_dotclock;
8039 }
8040
8041 static enum drm_mode_status
8042 intel_mode_valid(struct drm_device *dev,
8043                  const struct drm_display_mode *mode)
8044 {
8045         struct drm_i915_private *dev_priv = to_i915(dev);
8046         int hdisplay_max, htotal_max;
8047         int vdisplay_max, vtotal_max;
8048
8049         /*
8050          * Can't reject DBLSCAN here because Xorg ddxen can add piles
8051          * of DBLSCAN modes to the output's mode list when they detect
8052          * the scaling mode property on the connector. And they don't
8053          * ask the kernel to validate those modes in any way until
8054          * modeset time at which point the client gets a protocol error.
8055          * So in order to not upset those clients we silently ignore the
8056          * DBLSCAN flag on such connectors. For other connectors we will
8057          * reject modes with the DBLSCAN flag in encoder->compute_config().
8058          * And we always reject DBLSCAN modes in connector->mode_valid()
8059          * as we never want such modes on the connector's mode list.
8060          */
8061
8062         if (mode->vscan > 1)
8063                 return MODE_NO_VSCAN;
8064
8065         if (mode->flags & DRM_MODE_FLAG_HSKEW)
8066                 return MODE_H_ILLEGAL;
8067
8068         if (mode->flags & (DRM_MODE_FLAG_CSYNC |
8069                            DRM_MODE_FLAG_NCSYNC |
8070                            DRM_MODE_FLAG_PCSYNC))
8071                 return MODE_HSYNC;
8072
8073         if (mode->flags & (DRM_MODE_FLAG_BCAST |
8074                            DRM_MODE_FLAG_PIXMUX |
8075                            DRM_MODE_FLAG_CLKDIV2))
8076                 return MODE_BAD;
8077
8078         /*
8079          * Reject clearly excessive dotclocks early to
8080          * avoid having to worry about huge integers later.
8081          */
8082         if (mode->clock > max_dotclock(dev_priv))
8083                 return MODE_CLOCK_HIGH;
8084
8085         /* Transcoder timing limits */
8086         if (DISPLAY_VER(dev_priv) >= 11) {
8087                 hdisplay_max = 16384;
8088                 vdisplay_max = 8192;
8089                 htotal_max = 16384;
8090                 vtotal_max = 8192;
8091         } else if (DISPLAY_VER(dev_priv) >= 9 ||
8092                    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv)) {
8093                 hdisplay_max = 8192; /* FDI max 4096 handled elsewhere */
8094                 vdisplay_max = 4096;
8095                 htotal_max = 8192;
8096                 vtotal_max = 8192;
8097         } else if (DISPLAY_VER(dev_priv) >= 3) {
8098                 hdisplay_max = 4096;
8099                 vdisplay_max = 4096;
8100                 htotal_max = 8192;
8101                 vtotal_max = 8192;
8102         } else {
8103                 hdisplay_max = 2048;
8104                 vdisplay_max = 2048;
8105                 htotal_max = 4096;
8106                 vtotal_max = 4096;
8107         }
8108
8109         if (mode->hdisplay > hdisplay_max ||
8110             mode->hsync_start > htotal_max ||
8111             mode->hsync_end > htotal_max ||
8112             mode->htotal > htotal_max)
8113                 return MODE_H_ILLEGAL;
8114
8115         if (mode->vdisplay > vdisplay_max ||
8116             mode->vsync_start > vtotal_max ||
8117             mode->vsync_end > vtotal_max ||
8118             mode->vtotal > vtotal_max)
8119                 return MODE_V_ILLEGAL;
8120
8121         if (DISPLAY_VER(dev_priv) >= 5) {
8122                 if (mode->hdisplay < 64 ||
8123                     mode->htotal - mode->hdisplay < 32)
8124                         return MODE_H_ILLEGAL;
8125
8126                 if (mode->vtotal - mode->vdisplay < 5)
8127                         return MODE_V_ILLEGAL;
8128         } else {
8129                 if (mode->htotal - mode->hdisplay < 32)
8130                         return MODE_H_ILLEGAL;
8131
8132                 if (mode->vtotal - mode->vdisplay < 3)
8133                         return MODE_V_ILLEGAL;
8134         }
8135
8136         /*
8137          * Cantiga+ cannot handle modes with a hsync front porch of 0.
8138          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
8139          */
8140         if ((DISPLAY_VER(dev_priv) > 4 || IS_G4X(dev_priv)) &&
8141             mode->hsync_start == mode->hdisplay)
8142                 return MODE_H_ILLEGAL;
8143
8144         return MODE_OK;
8145 }
8146
8147 enum drm_mode_status
8148 intel_mode_valid_max_plane_size(struct drm_i915_private *dev_priv,
8149                                 const struct drm_display_mode *mode,
8150                                 bool bigjoiner)
8151 {
8152         int plane_width_max, plane_height_max;
8153
8154         /*
8155          * intel_mode_valid() should be
8156          * sufficient on older platforms.
8157          */
8158         if (DISPLAY_VER(dev_priv) < 9)
8159                 return MODE_OK;
8160
8161         /*
8162          * Most people will probably want a fullscreen
8163          * plane so let's not advertize modes that are
8164          * too big for that.
8165          */
8166         if (DISPLAY_VER(dev_priv) >= 11) {
8167                 plane_width_max = 5120 << bigjoiner;
8168                 plane_height_max = 4320;
8169         } else {
8170                 plane_width_max = 5120;
8171                 plane_height_max = 4096;
8172         }
8173
8174         if (mode->hdisplay > plane_width_max)
8175                 return MODE_H_ILLEGAL;
8176
8177         if (mode->vdisplay > plane_height_max)
8178                 return MODE_V_ILLEGAL;
8179
8180         return MODE_OK;
8181 }
8182
8183 static const struct drm_mode_config_funcs intel_mode_funcs = {
8184         .fb_create = intel_user_framebuffer_create,
8185         .get_format_info = intel_fb_get_format_info,
8186         .output_poll_changed = intel_fbdev_output_poll_changed,
8187         .mode_valid = intel_mode_valid,
8188         .atomic_check = intel_atomic_check,
8189         .atomic_commit = intel_atomic_commit,
8190         .atomic_state_alloc = intel_atomic_state_alloc,
8191         .atomic_state_clear = intel_atomic_state_clear,
8192         .atomic_state_free = intel_atomic_state_free,
8193 };
8194
8195 static const struct intel_display_funcs skl_display_funcs = {
8196         .get_pipe_config = hsw_get_pipe_config,
8197         .crtc_enable = hsw_crtc_enable,
8198         .crtc_disable = hsw_crtc_disable,
8199         .commit_modeset_enables = skl_commit_modeset_enables,
8200         .get_initial_plane_config = skl_get_initial_plane_config,
8201 };
8202
8203 static const struct intel_display_funcs ddi_display_funcs = {
8204         .get_pipe_config = hsw_get_pipe_config,
8205         .crtc_enable = hsw_crtc_enable,
8206         .crtc_disable = hsw_crtc_disable,
8207         .commit_modeset_enables = intel_commit_modeset_enables,
8208         .get_initial_plane_config = i9xx_get_initial_plane_config,
8209 };
8210
8211 static const struct intel_display_funcs pch_split_display_funcs = {
8212         .get_pipe_config = ilk_get_pipe_config,
8213         .crtc_enable = ilk_crtc_enable,
8214         .crtc_disable = ilk_crtc_disable,
8215         .commit_modeset_enables = intel_commit_modeset_enables,
8216         .get_initial_plane_config = i9xx_get_initial_plane_config,
8217 };
8218
8219 static const struct intel_display_funcs vlv_display_funcs = {
8220         .get_pipe_config = i9xx_get_pipe_config,
8221         .crtc_enable = valleyview_crtc_enable,
8222         .crtc_disable = i9xx_crtc_disable,
8223         .commit_modeset_enables = intel_commit_modeset_enables,
8224         .get_initial_plane_config = i9xx_get_initial_plane_config,
8225 };
8226
8227 static const struct intel_display_funcs i9xx_display_funcs = {
8228         .get_pipe_config = i9xx_get_pipe_config,
8229         .crtc_enable = i9xx_crtc_enable,
8230         .crtc_disable = i9xx_crtc_disable,
8231         .commit_modeset_enables = intel_commit_modeset_enables,
8232         .get_initial_plane_config = i9xx_get_initial_plane_config,
8233 };
8234
8235 /**
8236  * intel_init_display_hooks - initialize the display modesetting hooks
8237  * @dev_priv: device private
8238  */
8239 void intel_init_display_hooks(struct drm_i915_private *dev_priv)
8240 {
8241         if (!HAS_DISPLAY(dev_priv))
8242                 return;
8243
8244         intel_color_init_hooks(dev_priv);
8245         intel_init_cdclk_hooks(dev_priv);
8246         intel_audio_hooks_init(dev_priv);
8247
8248         intel_dpll_init_clock_hook(dev_priv);
8249
8250         if (DISPLAY_VER(dev_priv) >= 9) {
8251                 dev_priv->display.funcs.display = &skl_display_funcs;
8252         } else if (HAS_DDI(dev_priv)) {
8253                 dev_priv->display.funcs.display = &ddi_display_funcs;
8254         } else if (HAS_PCH_SPLIT(dev_priv)) {
8255                 dev_priv->display.funcs.display = &pch_split_display_funcs;
8256         } else if (IS_CHERRYVIEW(dev_priv) ||
8257                    IS_VALLEYVIEW(dev_priv)) {
8258                 dev_priv->display.funcs.display = &vlv_display_funcs;
8259         } else {
8260                 dev_priv->display.funcs.display = &i9xx_display_funcs;
8261         }
8262
8263         intel_fdi_init_hook(dev_priv);
8264 }
8265
8266 void intel_modeset_init_hw(struct drm_i915_private *i915)
8267 {
8268         struct intel_cdclk_state *cdclk_state;
8269
8270         if (!HAS_DISPLAY(i915))
8271                 return;
8272
8273         cdclk_state = to_intel_cdclk_state(i915->display.cdclk.obj.state);
8274
8275         intel_update_cdclk(i915);
8276         intel_cdclk_dump_config(i915, &i915->display.cdclk.hw, "Current CDCLK");
8277         cdclk_state->logical = cdclk_state->actual = i915->display.cdclk.hw;
8278 }
8279
8280 static int intel_initial_commit(struct drm_device *dev)
8281 {
8282         struct drm_atomic_state *state = NULL;
8283         struct drm_modeset_acquire_ctx ctx;
8284         struct intel_crtc *crtc;
8285         int ret = 0;
8286
8287         state = drm_atomic_state_alloc(dev);
8288         if (!state)
8289                 return -ENOMEM;
8290
8291         drm_modeset_acquire_init(&ctx, 0);
8292
8293 retry:
8294         state->acquire_ctx = &ctx;
8295
8296         for_each_intel_crtc(dev, crtc) {
8297                 struct intel_crtc_state *crtc_state =
8298                         intel_atomic_get_crtc_state(state, crtc);
8299
8300                 if (IS_ERR(crtc_state)) {
8301                         ret = PTR_ERR(crtc_state);
8302                         goto out;
8303                 }
8304
8305                 if (crtc_state->hw.active) {
8306                         struct intel_encoder *encoder;
8307
8308                         /*
8309                          * We've not yet detected sink capabilities
8310                          * (audio,infoframes,etc.) and thus we don't want to
8311                          * force a full state recomputation yet. We want that to
8312                          * happen only for the first real commit from userspace.
8313                          * So preserve the inherited flag for the time being.
8314                          */
8315                         crtc_state->inherited = true;
8316
8317                         ret = drm_atomic_add_affected_planes(state, &crtc->base);
8318                         if (ret)
8319                                 goto out;
8320
8321                         /*
8322                          * FIXME hack to force a LUT update to avoid the
8323                          * plane update forcing the pipe gamma on without
8324                          * having a proper LUT loaded. Remove once we
8325                          * have readout for pipe gamma enable.
8326                          */
8327                         crtc_state->uapi.color_mgmt_changed = true;
8328
8329                         for_each_intel_encoder_mask(dev, encoder,
8330                                                     crtc_state->uapi.encoder_mask) {
8331                                 if (encoder->initial_fastset_check &&
8332                                     !encoder->initial_fastset_check(encoder, crtc_state)) {
8333                                         ret = drm_atomic_add_affected_connectors(state,
8334                                                                                  &crtc->base);
8335                                         if (ret)
8336                                                 goto out;
8337                                 }
8338                         }
8339                 }
8340         }
8341
8342         ret = drm_atomic_commit(state);
8343
8344 out:
8345         if (ret == -EDEADLK) {
8346                 drm_atomic_state_clear(state);
8347                 drm_modeset_backoff(&ctx);
8348                 goto retry;
8349         }
8350
8351         drm_atomic_state_put(state);
8352
8353         drm_modeset_drop_locks(&ctx);
8354         drm_modeset_acquire_fini(&ctx);
8355
8356         return ret;
8357 }
8358
8359 static const struct drm_mode_config_helper_funcs intel_mode_config_funcs = {
8360         .atomic_commit_setup = drm_dp_mst_atomic_setup_commit,
8361 };
8362
8363 static void intel_mode_config_init(struct drm_i915_private *i915)
8364 {
8365         struct drm_mode_config *mode_config = &i915->drm.mode_config;
8366
8367         drm_mode_config_init(&i915->drm);
8368         INIT_LIST_HEAD(&i915->display.global.obj_list);
8369
8370         mode_config->min_width = 0;
8371         mode_config->min_height = 0;
8372
8373         mode_config->preferred_depth = 24;
8374         mode_config->prefer_shadow = 1;
8375
8376         mode_config->funcs = &intel_mode_funcs;
8377         mode_config->helper_private = &intel_mode_config_funcs;
8378
8379         mode_config->async_page_flip = HAS_ASYNC_FLIPS(i915);
8380
8381         /*
8382          * Maximum framebuffer dimensions, chosen to match
8383          * the maximum render engine surface size on gen4+.
8384          */
8385         if (DISPLAY_VER(i915) >= 7) {
8386                 mode_config->max_width = 16384;
8387                 mode_config->max_height = 16384;
8388         } else if (DISPLAY_VER(i915) >= 4) {
8389                 mode_config->max_width = 8192;
8390                 mode_config->max_height = 8192;
8391         } else if (DISPLAY_VER(i915) == 3) {
8392                 mode_config->max_width = 4096;
8393                 mode_config->max_height = 4096;
8394         } else {
8395                 mode_config->max_width = 2048;
8396                 mode_config->max_height = 2048;
8397         }
8398
8399         if (IS_I845G(i915) || IS_I865G(i915)) {
8400                 mode_config->cursor_width = IS_I845G(i915) ? 64 : 512;
8401                 mode_config->cursor_height = 1023;
8402         } else if (IS_I830(i915) || IS_I85X(i915) ||
8403                    IS_I915G(i915) || IS_I915GM(i915)) {
8404                 mode_config->cursor_width = 64;
8405                 mode_config->cursor_height = 64;
8406         } else {
8407                 mode_config->cursor_width = 256;
8408                 mode_config->cursor_height = 256;
8409         }
8410 }
8411
8412 static void intel_mode_config_cleanup(struct drm_i915_private *i915)
8413 {
8414         intel_atomic_global_obj_cleanup(i915);
8415         drm_mode_config_cleanup(&i915->drm);
8416 }
8417
8418 /* part #1: call before irq install */
8419 int intel_modeset_init_noirq(struct drm_i915_private *i915)
8420 {
8421         int ret;
8422
8423         if (i915_inject_probe_failure(i915))
8424                 return -ENODEV;
8425
8426         if (HAS_DISPLAY(i915)) {
8427                 ret = drm_vblank_init(&i915->drm,
8428                                       INTEL_NUM_PIPES(i915));
8429                 if (ret)
8430                         return ret;
8431         }
8432
8433         intel_bios_init(i915);
8434
8435         ret = intel_vga_register(i915);
8436         if (ret)
8437                 goto cleanup_bios;
8438
8439         /* FIXME: completely on the wrong abstraction layer */
8440         ret = intel_power_domains_init(i915);
8441         if (ret < 0)
8442                 goto cleanup_vga;
8443
8444         intel_power_domains_init_hw(i915, false);
8445
8446         if (!HAS_DISPLAY(i915))
8447                 return 0;
8448
8449         intel_dmc_init(i915);
8450
8451         i915->display.wq.modeset = alloc_ordered_workqueue("i915_modeset", 0);
8452         i915->display.wq.flip = alloc_workqueue("i915_flip", WQ_HIGHPRI |
8453                                                 WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
8454
8455         intel_mode_config_init(i915);
8456
8457         ret = intel_cdclk_init(i915);
8458         if (ret)
8459                 goto cleanup_vga_client_pw_domain_dmc;
8460
8461         ret = intel_color_init(i915);
8462         if (ret)
8463                 goto cleanup_vga_client_pw_domain_dmc;
8464
8465         ret = intel_dbuf_init(i915);
8466         if (ret)
8467                 goto cleanup_vga_client_pw_domain_dmc;
8468
8469         ret = intel_bw_init(i915);
8470         if (ret)
8471                 goto cleanup_vga_client_pw_domain_dmc;
8472
8473         init_llist_head(&i915->display.atomic_helper.free_list);
8474         INIT_WORK(&i915->display.atomic_helper.free_work,
8475                   intel_atomic_helper_free_state_worker);
8476
8477         intel_init_quirks(i915);
8478
8479         intel_fbc_init(i915);
8480
8481         return 0;
8482
8483 cleanup_vga_client_pw_domain_dmc:
8484         intel_dmc_fini(i915);
8485         intel_power_domains_driver_remove(i915);
8486 cleanup_vga:
8487         intel_vga_unregister(i915);
8488 cleanup_bios:
8489         intel_bios_driver_remove(i915);
8490
8491         return ret;
8492 }
8493
8494 /* part #2: call after irq install, but before gem init */
8495 int intel_modeset_init_nogem(struct drm_i915_private *i915)
8496 {
8497         struct drm_device *dev = &i915->drm;
8498         enum pipe pipe;
8499         struct intel_crtc *crtc;
8500         int ret;
8501
8502         if (!HAS_DISPLAY(i915))
8503                 return 0;
8504
8505         intel_wm_init(i915);
8506
8507         intel_panel_sanitize_ssc(i915);
8508
8509         intel_pps_setup(i915);
8510
8511         intel_gmbus_setup(i915);
8512
8513         drm_dbg_kms(&i915->drm, "%d display pipe%s available.\n",
8514                     INTEL_NUM_PIPES(i915),
8515                     INTEL_NUM_PIPES(i915) > 1 ? "s" : "");
8516
8517         for_each_pipe(i915, pipe) {
8518                 ret = intel_crtc_init(i915, pipe);
8519                 if (ret) {
8520                         intel_mode_config_cleanup(i915);
8521                         return ret;
8522                 }
8523         }
8524
8525         intel_plane_possible_crtcs_init(i915);
8526         intel_shared_dpll_init(i915);
8527         intel_fdi_pll_freq_update(i915);
8528
8529         intel_update_czclk(i915);
8530         intel_modeset_init_hw(i915);
8531         intel_dpll_update_ref_clks(i915);
8532
8533         intel_hdcp_component_init(i915);
8534
8535         if (i915->display.cdclk.max_cdclk_freq == 0)
8536                 intel_update_max_cdclk(i915);
8537
8538         intel_hti_init(i915);
8539
8540         /* Just disable it once at startup */
8541         intel_vga_disable(i915);
8542         intel_setup_outputs(i915);
8543
8544         drm_modeset_lock_all(dev);
8545         intel_modeset_setup_hw_state(i915, dev->mode_config.acquire_ctx);
8546         intel_acpi_assign_connector_fwnodes(i915);
8547         drm_modeset_unlock_all(dev);
8548
8549         for_each_intel_crtc(dev, crtc) {
8550                 if (!to_intel_crtc_state(crtc->base.state)->uapi.active)
8551                         continue;
8552                 intel_crtc_initial_plane_config(crtc);
8553         }
8554
8555         /*
8556          * Make sure hardware watermarks really match the state we read out.
8557          * Note that we need to do this after reconstructing the BIOS fb's
8558          * since the watermark calculation done here will use pstate->fb.
8559          */
8560         if (!HAS_GMCH(i915))
8561                 ilk_wm_sanitize(i915);
8562
8563         return 0;
8564 }
8565
8566 /* part #3: call after gem init */
8567 int intel_modeset_init(struct drm_i915_private *i915)
8568 {
8569         int ret;
8570
8571         if (!HAS_DISPLAY(i915))
8572                 return 0;
8573
8574         /*
8575          * Force all active planes to recompute their states. So that on
8576          * mode_setcrtc after probe, all the intel_plane_state variables
8577          * are already calculated and there is no assert_plane warnings
8578          * during bootup.
8579          */
8580         ret = intel_initial_commit(&i915->drm);
8581         if (ret)
8582                 drm_dbg_kms(&i915->drm, "Initial modeset failed, %d\n", ret);
8583
8584         intel_overlay_setup(i915);
8585
8586         ret = intel_fbdev_init(&i915->drm);
8587         if (ret)
8588                 return ret;
8589
8590         /* Only enable hotplug handling once the fbdev is fully set up. */
8591         intel_hpd_init(i915);
8592         intel_hpd_poll_disable(i915);
8593
8594         skl_watermark_ipc_init(i915);
8595
8596         return 0;
8597 }
8598
8599 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
8600 {
8601         struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv, pipe);
8602         enum transcoder cpu_transcoder = (enum transcoder)pipe;
8603         /* 640x480@60Hz, ~25175 kHz */
8604         struct dpll clock = {
8605                 .m1 = 18,
8606                 .m2 = 7,
8607                 .p1 = 13,
8608                 .p2 = 4,
8609                 .n = 2,
8610         };
8611         u32 dpll, fp;
8612         int i;
8613
8614         drm_WARN_ON(&dev_priv->drm,
8615                     i9xx_calc_dpll_params(48000, &clock) != 25154);
8616
8617         drm_dbg_kms(&dev_priv->drm,
8618                     "enabling pipe %c due to force quirk (vco=%d dot=%d)\n",
8619                     pipe_name(pipe), clock.vco, clock.dot);
8620
8621         fp = i9xx_dpll_compute_fp(&clock);
8622         dpll = DPLL_DVO_2X_MODE |
8623                 DPLL_VGA_MODE_DIS |
8624                 ((clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT) |
8625                 PLL_P2_DIVIDE_BY_4 |
8626                 PLL_REF_INPUT_DREFCLK |
8627                 DPLL_VCO_ENABLE;
8628
8629         intel_de_write(dev_priv, TRANS_HTOTAL(cpu_transcoder),
8630                        HACTIVE(640 - 1) | HTOTAL(800 - 1));
8631         intel_de_write(dev_priv, TRANS_HBLANK(cpu_transcoder),
8632                        HBLANK_START(640 - 1) | HBLANK_END(800 - 1));
8633         intel_de_write(dev_priv, TRANS_HSYNC(cpu_transcoder),
8634                        HSYNC_START(656 - 1) | HSYNC_END(752 - 1));
8635         intel_de_write(dev_priv, TRANS_VTOTAL(cpu_transcoder),
8636                        VACTIVE(480 - 1) | VTOTAL(525 - 1));
8637         intel_de_write(dev_priv, TRANS_VBLANK(cpu_transcoder),
8638                        VBLANK_START(480 - 1) | VBLANK_END(525 - 1));
8639         intel_de_write(dev_priv, TRANS_VSYNC(cpu_transcoder),
8640                        VSYNC_START(490 - 1) | VSYNC_END(492 - 1));
8641         intel_de_write(dev_priv, PIPESRC(pipe),
8642                        PIPESRC_WIDTH(640 - 1) | PIPESRC_HEIGHT(480 - 1));
8643
8644         intel_de_write(dev_priv, FP0(pipe), fp);
8645         intel_de_write(dev_priv, FP1(pipe), fp);
8646
8647         /*
8648          * Apparently we need to have VGA mode enabled prior to changing
8649          * the P1/P2 dividers. Otherwise the DPLL will keep using the old
8650          * dividers, even though the register value does change.
8651          */
8652         intel_de_write(dev_priv, DPLL(pipe), dpll & ~DPLL_VGA_MODE_DIS);
8653         intel_de_write(dev_priv, DPLL(pipe), dpll);
8654
8655         /* Wait for the clocks to stabilize. */
8656         intel_de_posting_read(dev_priv, DPLL(pipe));
8657         udelay(150);
8658
8659         /* The pixel multiplier can only be updated once the
8660          * DPLL is enabled and the clocks are stable.
8661          *
8662          * So write it again.
8663          */
8664         intel_de_write(dev_priv, DPLL(pipe), dpll);
8665
8666         /* We do this three times for luck */
8667         for (i = 0; i < 3 ; i++) {
8668                 intel_de_write(dev_priv, DPLL(pipe), dpll);
8669                 intel_de_posting_read(dev_priv, DPLL(pipe));
8670                 udelay(150); /* wait for warmup */
8671         }
8672
8673         intel_de_write(dev_priv, TRANSCONF(pipe), TRANSCONF_ENABLE);
8674         intel_de_posting_read(dev_priv, TRANSCONF(pipe));
8675
8676         intel_wait_for_pipe_scanline_moving(crtc);
8677 }
8678
8679 void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
8680 {
8681         struct intel_crtc *crtc = intel_crtc_for_pipe(dev_priv, pipe);
8682
8683         drm_dbg_kms(&dev_priv->drm, "disabling pipe %c due to force quirk\n",
8684                     pipe_name(pipe));
8685
8686         drm_WARN_ON(&dev_priv->drm,
8687                     intel_de_read(dev_priv, DSPCNTR(PLANE_A)) & DISP_ENABLE);
8688         drm_WARN_ON(&dev_priv->drm,
8689                     intel_de_read(dev_priv, DSPCNTR(PLANE_B)) & DISP_ENABLE);
8690         drm_WARN_ON(&dev_priv->drm,
8691                     intel_de_read(dev_priv, DSPCNTR(PLANE_C)) & DISP_ENABLE);
8692         drm_WARN_ON(&dev_priv->drm,
8693                     intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE_MASK);
8694         drm_WARN_ON(&dev_priv->drm,
8695                     intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE_MASK);
8696
8697         intel_de_write(dev_priv, TRANSCONF(pipe), 0);
8698         intel_de_posting_read(dev_priv, TRANSCONF(pipe));
8699
8700         intel_wait_for_pipe_scanline_stopped(crtc);
8701
8702         intel_de_write(dev_priv, DPLL(pipe), DPLL_VGA_MODE_DIS);
8703         intel_de_posting_read(dev_priv, DPLL(pipe));
8704 }
8705
8706 void intel_display_resume(struct drm_device *dev)
8707 {
8708         struct drm_i915_private *i915 = to_i915(dev);
8709         struct drm_atomic_state *state = i915->display.restore.modeset_state;
8710         struct drm_modeset_acquire_ctx ctx;
8711         int ret;
8712
8713         if (!HAS_DISPLAY(i915))
8714                 return;
8715
8716         i915->display.restore.modeset_state = NULL;
8717         if (state)
8718                 state->acquire_ctx = &ctx;
8719
8720         drm_modeset_acquire_init(&ctx, 0);
8721
8722         while (1) {
8723                 ret = drm_modeset_lock_all_ctx(dev, &ctx);
8724                 if (ret != -EDEADLK)
8725                         break;
8726
8727                 drm_modeset_backoff(&ctx);
8728         }
8729
8730         if (!ret)
8731                 ret = __intel_display_resume(i915, state, &ctx);
8732
8733         skl_watermark_ipc_update(i915);
8734         drm_modeset_drop_locks(&ctx);
8735         drm_modeset_acquire_fini(&ctx);
8736
8737         if (ret)
8738                 drm_err(&i915->drm,
8739                         "Restoring old state failed with %i\n", ret);
8740         if (state)
8741                 drm_atomic_state_put(state);
8742 }
8743
8744 static void intel_hpd_poll_fini(struct drm_i915_private *i915)
8745 {
8746         struct intel_connector *connector;
8747         struct drm_connector_list_iter conn_iter;
8748
8749         /* Kill all the work that may have been queued by hpd. */
8750         drm_connector_list_iter_begin(&i915->drm, &conn_iter);
8751         for_each_intel_connector_iter(connector, &conn_iter) {
8752                 if (connector->modeset_retry_work.func)
8753                         cancel_work_sync(&connector->modeset_retry_work);
8754                 if (connector->hdcp.shim) {
8755                         cancel_delayed_work_sync(&connector->hdcp.check_work);
8756                         cancel_work_sync(&connector->hdcp.prop_work);
8757                 }
8758         }
8759         drm_connector_list_iter_end(&conn_iter);
8760 }
8761
8762 /* part #1: call before irq uninstall */
8763 void intel_modeset_driver_remove(struct drm_i915_private *i915)
8764 {
8765         if (!HAS_DISPLAY(i915))
8766                 return;
8767
8768         flush_workqueue(i915->display.wq.flip);
8769         flush_workqueue(i915->display.wq.modeset);
8770
8771         flush_work(&i915->display.atomic_helper.free_work);
8772         drm_WARN_ON(&i915->drm, !llist_empty(&i915->display.atomic_helper.free_list));
8773
8774         /*
8775          * MST topology needs to be suspended so we don't have any calls to
8776          * fbdev after it's finalized. MST will be destroyed later as part of
8777          * drm_mode_config_cleanup()
8778          */
8779         intel_dp_mst_suspend(i915);
8780 }
8781
8782 /* part #2: call after irq uninstall */
8783 void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
8784 {
8785         if (!HAS_DISPLAY(i915))
8786                 return;
8787
8788         /*
8789          * Due to the hpd irq storm handling the hotplug work can re-arm the
8790          * poll handlers. Hence disable polling after hpd handling is shut down.
8791          */
8792         intel_hpd_poll_fini(i915);
8793
8794         /* poll work can call into fbdev, hence clean that up afterwards */
8795         intel_fbdev_fini(i915);
8796
8797         intel_unregister_dsm_handler();
8798
8799         /* flush any delayed tasks or pending work */
8800         flush_scheduled_work();
8801
8802         intel_hdcp_component_fini(i915);
8803
8804         intel_mode_config_cleanup(i915);
8805
8806         intel_overlay_cleanup(i915);
8807
8808         intel_gmbus_teardown(i915);
8809
8810         destroy_workqueue(i915->display.wq.flip);
8811         destroy_workqueue(i915->display.wq.modeset);
8812
8813         intel_fbc_cleanup(i915);
8814 }
8815
8816 /* part #3: call after gem init */
8817 void intel_modeset_driver_remove_nogem(struct drm_i915_private *i915)
8818 {
8819         intel_dmc_fini(i915);
8820
8821         intel_power_domains_driver_remove(i915);
8822
8823         intel_vga_unregister(i915);
8824
8825         intel_bios_driver_remove(i915);
8826 }
8827
8828 bool intel_modeset_probe_defer(struct pci_dev *pdev)
8829 {
8830         struct drm_privacy_screen *privacy_screen;
8831
8832         /*
8833          * apple-gmux is needed on dual GPU MacBook Pro
8834          * to probe the panel if we're the inactive GPU.
8835          */
8836         if (vga_switcheroo_client_probe_defer(pdev))
8837                 return true;
8838
8839         /* If the LCD panel has a privacy-screen, wait for it */
8840         privacy_screen = drm_privacy_screen_get(&pdev->dev, NULL);
8841         if (IS_ERR(privacy_screen) && PTR_ERR(privacy_screen) == -EPROBE_DEFER)
8842                 return true;
8843
8844         drm_privacy_screen_put(privacy_screen);
8845
8846         return false;
8847 }
8848
8849 void intel_display_driver_register(struct drm_i915_private *i915)
8850 {
8851         if (!HAS_DISPLAY(i915))
8852                 return;
8853
8854         /* Must be done after probing outputs */
8855         intel_opregion_register(i915);
8856         intel_acpi_video_register(i915);
8857
8858         intel_audio_init(i915);
8859
8860         intel_display_debugfs_register(i915);
8861
8862         /*
8863          * Some ports require correctly set-up hpd registers for
8864          * detection to work properly (leading to ghost connected
8865          * connector status), e.g. VGA on gm45.  Hence we can only set
8866          * up the initial fbdev config after hpd irqs are fully
8867          * enabled. We do it last so that the async config cannot run
8868          * before the connectors are registered.
8869          */
8870         intel_fbdev_initial_config_async(i915);
8871
8872         /*
8873          * We need to coordinate the hotplugs with the asynchronous
8874          * fbdev configuration, for which we use the
8875          * fbdev->async_cookie.
8876          */
8877         drm_kms_helper_poll_init(&i915->drm);
8878 }
8879
8880 void intel_display_driver_unregister(struct drm_i915_private *i915)
8881 {
8882         if (!HAS_DISPLAY(i915))
8883                 return;
8884
8885         intel_fbdev_unregister(i915);
8886         intel_audio_deinit(i915);
8887
8888         /*
8889          * After flushing the fbdev (incl. a late async config which
8890          * will have delayed queuing of a hotplug event), then flush
8891          * the hotplug events.
8892          */
8893         drm_kms_helper_poll_fini(&i915->drm);
8894         drm_atomic_helper_shutdown(&i915->drm);
8895
8896         acpi_video_unregister();
8897         intel_opregion_unregister(i915);
8898 }
8899
8900 bool intel_scanout_needs_vtd_wa(struct drm_i915_private *i915)
8901 {
8902         return DISPLAY_VER(i915) >= 6 && i915_vtd_active(i915);
8903 }