drivers/gpu/drm/i915/display/intel_psr.c

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <drm/drm_atomic_helper.h>
  25
  26 #include "display/intel_dp.h"
  27
  28 #include "i915_drv.h"
  29 #include "intel_atomic.h"
  30 #include "intel_display_types.h"
  31 #include "intel_psr.h"
  32 #include "intel_sprite.h"
  33
  34 /**
  35  * DOC: Panel Self Refresh (PSR/SRD)
  36  *
  37  * Since Haswell Display controller supports Panel Self-Refresh on display
  38  * panels witch have a remote frame buffer (RFB) implemented according to PSR
  39  * spec in eDP1.3. PSR feature allows the display to go to lower standby states
  40  * when system is idle but display is on as it eliminates display refresh
  41  * request to DDR memory completely as long as the frame buffer for that
  42  * display is unchanged.
  43  *
  44  * Panel Self Refresh must be supported by both Hardware (source) and
  45  * Panel (sink).
  46  *
  47  * PSR saves power by caching the framebuffer in the panel RFB, which allows us
  48  * to power down the link and memory controller. For DSI panels the same idea
  49  * is called "manual mode".
  50  *
  51  * The implementation uses the hardware-based PSR support which automatically
  52  * enters/exits self-refresh mode. The hardware takes care of sending the
  53  * required DP aux message and could even retrain the link (that part isn't
  54  * enabled yet though). The hardware also keeps track of any frontbuffer
  55  * changes to know when to exit self-refresh mode again. Unfortunately that
  56  * part doesn't work too well, hence why the i915 PSR support uses the
  57  * software frontbuffer tracking to make sure it doesn't miss a screen
  58  * update. For this integration intel_psr_invalidate() and intel_psr_flush()
  59  * get called by the frontbuffer tracking code. Note that because of locking
  60  * issues the self-refresh re-enable code is done from a work queue, which
  61  * must be correctly synchronized/cancelled when shutting down the pipe."
  62  */
  63
  64 static bool psr_global_enabled(u32 debug)
  65 {
  66         switch (debug & I915_PSR_DEBUG_MODE_MASK) {
  67         case I915_PSR_DEBUG_DEFAULT:
  68                 return i915_modparams.enable_psr;
  69         case I915_PSR_DEBUG_DISABLE:
  70                 return false;
  71         default:
  72                 return true;
  73         }
  74 }
  75
  76 static bool intel_psr2_enabled(struct drm_i915_private *dev_priv,
  77                                const struct intel_crtc_state *crtc_state)
  78 {
  79         /* Cannot enable DSC and PSR2 simultaneously */
  80         WARN_ON(crtc_state->dsc.compression_enable &&
  81                 crtc_state->has_psr2);
  82
  83         switch (dev_priv->psr.debug & I915_PSR_DEBUG_MODE_MASK) {
  84         case I915_PSR_DEBUG_DISABLE:
  85         case I915_PSR_DEBUG_FORCE_PSR1:
  86                 return false;
  87         default:
  88                 return crtc_state->has_psr2;
  89         }
  90 }
  91
  92 static void psr_irq_control(struct drm_i915_private *dev_priv)
  93 {
  94         enum transcoder trans_shift;
  95         u32 mask, val;
  96         i915_reg_t imr_reg;
  97
  98         /*
  99          * gen12+ has registers relative to transcoder and one per transcoder
 100          * using the same bit definition: handle it as TRANSCODER_EDP to force
 101          * 0 shift in bit definition
 102          */
 103         if (INTEL_GEN(dev_priv) >= 12) {
 104                 trans_shift = 0;
 105                 imr_reg = TRANS_PSR_IMR(dev_priv->psr.transcoder);
 106         } else {
 107                 trans_shift = dev_priv->psr.transcoder;
 108                 imr_reg = EDP_PSR_IMR;
 109         }
 110
 111         mask = EDP_PSR_ERROR(trans_shift);
 112         if (dev_priv->psr.debug & I915_PSR_DEBUG_IRQ)
 113                 mask |= EDP_PSR_POST_EXIT(trans_shift) |
 114                         EDP_PSR_PRE_ENTRY(trans_shift);
 115
 116         /* Warning: it is masking/setting reserved bits too */
 117         val = I915_READ(imr_reg);
 118         val &= ~EDP_PSR_TRANS_MASK(trans_shift);
 119         val |= ~mask;
 120         I915_WRITE(imr_reg, val);
 121 }
 122
 123 static void psr_event_print(u32 val, bool psr2_enabled)
 124 {
 125         DRM_DEBUG_KMS("PSR exit events: 0x%x\n", val);
 126         if (val & PSR_EVENT_PSR2_WD_TIMER_EXPIRE)
 127                 DRM_DEBUG_KMS("\tPSR2 watchdog timer expired\n");
 128         if ((val & PSR_EVENT_PSR2_DISABLED) && psr2_enabled)
 129                 DRM_DEBUG_KMS("\tPSR2 disabled\n");
 130         if (val & PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN)
 131                 DRM_DEBUG_KMS("\tSU dirty FIFO underrun\n");
 132         if (val & PSR_EVENT_SU_CRC_FIFO_UNDERRUN)
 133                 DRM_DEBUG_KMS("\tSU CRC FIFO underrun\n");
 134         if (val & PSR_EVENT_GRAPHICS_RESET)
 135                 DRM_DEBUG_KMS("\tGraphics reset\n");
 136         if (val & PSR_EVENT_PCH_INTERRUPT)
 137                 DRM_DEBUG_KMS("\tPCH interrupt\n");
 138         if (val & PSR_EVENT_MEMORY_UP)
 139                 DRM_DEBUG_KMS("\tMemory up\n");
 140         if (val & PSR_EVENT_FRONT_BUFFER_MODIFY)
 141                 DRM_DEBUG_KMS("\tFront buffer modification\n");
 142         if (val & PSR_EVENT_WD_TIMER_EXPIRE)
 143                 DRM_DEBUG_KMS("\tPSR watchdog timer expired\n");
 144         if (val & PSR_EVENT_PIPE_REGISTERS_UPDATE)
 145                 DRM_DEBUG_KMS("\tPIPE registers updated\n");
 146         if (val & PSR_EVENT_REGISTER_UPDATE)
 147                 DRM_DEBUG_KMS("\tRegister updated\n");
 148         if (val & PSR_EVENT_HDCP_ENABLE)
 149                 DRM_DEBUG_KMS("\tHDCP enabled\n");
 150         if (val & PSR_EVENT_KVMR_SESSION_ENABLE)
 151                 DRM_DEBUG_KMS("\tKVMR session enabled\n");
 152         if (val & PSR_EVENT_VBI_ENABLE)
 153                 DRM_DEBUG_KMS("\tVBI enabled\n");
 154         if (val & PSR_EVENT_LPSP_MODE_EXIT)
 155                 DRM_DEBUG_KMS("\tLPSP mode exited\n");
 156         if ((val & PSR_EVENT_PSR_DISABLE) && !psr2_enabled)
 157                 DRM_DEBUG_KMS("\tPSR disabled\n");
 158 }
 159
 160 void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir)
 161 {
 162         enum transcoder cpu_transcoder = dev_priv->psr.transcoder;
 163         enum transcoder trans_shift;
 164         i915_reg_t imr_reg;
 165         ktime_t time_ns =  ktime_get();
 166
 167         if (INTEL_GEN(dev_priv) >= 12) {
 168                 trans_shift = 0;
 169                 imr_reg = TRANS_PSR_IMR(dev_priv->psr.transcoder);
 170         } else {
 171                 trans_shift = dev_priv->psr.transcoder;
 172                 imr_reg = EDP_PSR_IMR;
 173         }
 174
 175         if (psr_iir & EDP_PSR_PRE_ENTRY(trans_shift)) {
 176                 dev_priv->psr.last_entry_attempt = time_ns;
 177                 DRM_DEBUG_KMS("[transcoder %s] PSR entry attempt in 2 vblanks\n",
 178                               transcoder_name(cpu_transcoder));
 179         }
 180
 181         if (psr_iir & EDP_PSR_POST_EXIT(trans_shift)) {
 182                 dev_priv->psr.last_exit = time_ns;
 183                 DRM_DEBUG_KMS("[transcoder %s] PSR exit completed\n",
 184                               transcoder_name(cpu_transcoder));
 185
 186                 if (INTEL_GEN(dev_priv) >= 9) {
 187                         u32 val = I915_READ(PSR_EVENT(cpu_transcoder));
 188                         bool psr2_enabled = dev_priv->psr.psr2_enabled;
 189
 190                         I915_WRITE(PSR_EVENT(cpu_transcoder), val);
 191                         psr_event_print(val, psr2_enabled);
 192                 }
 193         }
 194
 195         if (psr_iir & EDP_PSR_ERROR(trans_shift)) {
 196                 u32 val;
 197
 198                 DRM_WARN("[transcoder %s] PSR aux error\n",
 199                          transcoder_name(cpu_transcoder));
 200
 201                 dev_priv->psr.irq_aux_error = true;
 202
 203                 /*
 204                  * If this interruption is not masked it will keep
 205                  * interrupting so fast that it prevents the scheduled
 206                  * work to run.
 207                  * Also after a PSR error, we don't want to arm PSR
 208                  * again so we don't care about unmask the interruption
 209                  * or unset irq_aux_error.
 210                  */
 211                 val = I915_READ(imr_reg);
 212                 val |= EDP_PSR_ERROR(trans_shift);
 213                 I915_WRITE(imr_reg, val);
 214
 215                 schedule_work(&dev_priv->psr.work);
 216         }
 217 }
 218
 219 static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp)
 220 {
 221         u8 alpm_caps = 0;
 222
 223         if (drm_dp_dpcd_readb(&intel_dp->aux, DP_RECEIVER_ALPM_CAP,
 224                               &alpm_caps) != 1)
 225                 return false;
 226         return alpm_caps & DP_ALPM_CAP;
 227 }
 228
 229 static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp)
 230 {
 231         u8 val = 8; /* assume the worst if we can't read the value */
 232
 233         if (drm_dp_dpcd_readb(&intel_dp->aux,
 234                               DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val) == 1)
 235                 val &= DP_MAX_RESYNC_FRAME_COUNT_MASK;
 236         else
 237                 DRM_DEBUG_KMS("Unable to get sink synchronization latency, assuming 8 frames\n");
 238         return val;
 239 }
 240
 241 static u16 intel_dp_get_su_x_granulartiy(struct intel_dp *intel_dp)
 242 {
 243         u16 val;
 244         ssize_t r;
 245
 246         /*
 247          * Returning the default X granularity if granularity not required or
 248          * if DPCD read fails
 249          */
 250         if (!(intel_dp->psr_dpcd[1] & DP_PSR2_SU_GRANULARITY_REQUIRED))
 251                 return 4;
 252
 253         r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &val, 2);
 254         if (r != 2)
 255                 DRM_DEBUG_KMS("Unable to read DP_PSR2_SU_X_GRANULARITY\n");
 256
 257         /*
 258          * Spec says that if the value read is 0 the default granularity should
 259          * be used instead.
 260          */
 261         if (r != 2 || val == 0)
 262                 val = 4;
 263
 264         return val;
 265 }
 266
 267 void intel_psr_init_dpcd(struct intel_dp *intel_dp)
 268 {
 269         struct drm_i915_private *dev_priv =
 270                 to_i915(dp_to_dig_port(intel_dp)->base.base.dev);
 271
 272         if (dev_priv->psr.dp) {
 273                 DRM_WARN("More than one eDP panel found, PSR support should be extended\n");
 274                 return;
 275         }
 276
 277         drm_dp_dpcd_read(&intel_dp->aux, DP_PSR_SUPPORT, intel_dp->psr_dpcd,
 278                          sizeof(intel_dp->psr_dpcd));
 279
 280         if (!intel_dp->psr_dpcd[0])
 281                 return;
 282         DRM_DEBUG_KMS("eDP panel supports PSR version %x\n",
 283                       intel_dp->psr_dpcd[0]);
 284
 285         if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) {
 286                 DRM_DEBUG_KMS("PSR support not currently available for this panel\n");
 287                 return;
 288         }
 289
 290         if (!(intel_dp->edp_dpcd[1] & DP_EDP_SET_POWER_CAP)) {
 291                 DRM_DEBUG_KMS("Panel lacks power state control, PSR cannot be enabled\n");
 292                 return;
 293         }
 294
 295         dev_priv->psr.sink_support = true;
 296         dev_priv->psr.sink_sync_latency =
 297                 intel_dp_get_sink_sync_latency(intel_dp);
 298
 299         dev_priv->psr.dp = intel_dp;
 300
 301         if (INTEL_GEN(dev_priv) >= 9 &&
 302             (intel_dp->psr_dpcd[0] == DP_PSR2_WITH_Y_COORD_IS_SUPPORTED)) {
 303                 bool y_req = intel_dp->psr_dpcd[1] &
 304                              DP_PSR2_SU_Y_COORDINATE_REQUIRED;
 305                 bool alpm = intel_dp_get_alpm_status(intel_dp);
 306
 307                 /*
 308                  * All panels that supports PSR version 03h (PSR2 +
 309                  * Y-coordinate) can handle Y-coordinates in VSC but we are
 310                  * only sure that it is going to be used when required by the
 311                  * panel. This way panel is capable to do selective update
 312                  * without a aux frame sync.
 313                  *
 314                  * To support PSR version 02h and PSR version 03h without
 315                  * Y-coordinate requirement panels we would need to enable
 316                  * GTC first.
 317                  */
 318                 dev_priv->psr.sink_psr2_support = y_req && alpm;
 319                 DRM_DEBUG_KMS("PSR2 %ssupported\n",
 320                               dev_priv->psr.sink_psr2_support ? "" : "not ");
 321
 322                 if (dev_priv->psr.sink_psr2_support) {
 323                         dev_priv->psr.colorimetry_support =
 324                                 intel_dp_get_colorimetry_status(intel_dp);
 325                         dev_priv->psr.su_x_granularity =
 326                                 intel_dp_get_su_x_granulartiy(intel_dp);
 327                 }
 328         }
 329 }
 330
 331 static void intel_psr_setup_vsc(struct intel_dp *intel_dp,
 332                                 const struct intel_crtc_state *crtc_state)
 333 {
 334         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
 335         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 336         struct dp_sdp psr_vsc;
 337
 338         if (dev_priv->psr.psr2_enabled) {
 339                 /* Prepare VSC Header for SU as per EDP 1.4 spec, Table 6.11 */
 340                 memset(&psr_vsc, 0, sizeof(psr_vsc));
 341                 psr_vsc.sdp_header.HB0 = 0;
 342                 psr_vsc.sdp_header.HB1 = 0x7;
 343                 if (dev_priv->psr.colorimetry_support) {
 344                         psr_vsc.sdp_header.HB2 = 0x5;
 345                         psr_vsc.sdp_header.HB3 = 0x13;
 346                 } else {
 347                         psr_vsc.sdp_header.HB2 = 0x4;
 348                         psr_vsc.sdp_header.HB3 = 0xe;
 349                 }
 350         } else {
 351                 /* Prepare VSC packet as per EDP 1.3 spec, Table 3.10 */
 352                 memset(&psr_vsc, 0, sizeof(psr_vsc));
 353                 psr_vsc.sdp_header.HB0 = 0;
 354                 psr_vsc.sdp_header.HB1 = 0x7;
 355                 psr_vsc.sdp_header.HB2 = 0x2;
 356                 psr_vsc.sdp_header.HB3 = 0x8;
 357         }
 358
 359         intel_dig_port->write_infoframe(&intel_dig_port->base,
 360                                         crtc_state,
 361                                         DP_SDP_VSC, &psr_vsc, sizeof(psr_vsc));
 362 }
 363
 364 static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
 365 {
 366         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 367         u32 aux_clock_divider, aux_ctl;
 368         int i;
 369         static const u8 aux_msg[] = {
 370                 [0] = DP_AUX_NATIVE_WRITE << 4,
 371                 [1] = DP_SET_POWER >> 8,
 372                 [2] = DP_SET_POWER & 0xff,
 373                 [3] = 1 - 1,
 374                 [4] = DP_SET_POWER_D0,
 375         };
 376         u32 psr_aux_mask = EDP_PSR_AUX_CTL_TIME_OUT_MASK |
 377                            EDP_PSR_AUX_CTL_MESSAGE_SIZE_MASK |
 378                            EDP_PSR_AUX_CTL_PRECHARGE_2US_MASK |
 379                            EDP_PSR_AUX_CTL_BIT_CLOCK_2X_MASK;
 380
 381         BUILD_BUG_ON(sizeof(aux_msg) > 20);
 382         for (i = 0; i < sizeof(aux_msg); i += 4)
 383                 I915_WRITE(EDP_PSR_AUX_DATA(dev_priv->psr.transcoder, i >> 2),
 384                            intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i));
 385
 386         aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
 387
 388         /* Start with bits set for DDI_AUX_CTL register */
 389         aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, sizeof(aux_msg),
 390                                              aux_clock_divider);
 391
 392         /* Select only valid bits for SRD_AUX_CTL */
 393         aux_ctl &= psr_aux_mask;
 394         I915_WRITE(EDP_PSR_AUX_CTL(dev_priv->psr.transcoder), aux_ctl);
 395 }
 396
 397 static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 398 {
 399         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 400         u8 dpcd_val = DP_PSR_ENABLE;
 401
 402         /* Enable ALPM at sink for psr2 */
 403         if (dev_priv->psr.psr2_enabled) {
 404                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG,
 405                                    DP_ALPM_ENABLE |
 406                                    DP_ALPM_LOCK_ERROR_IRQ_HPD_ENABLE);
 407
 408                 dpcd_val |= DP_PSR_ENABLE_PSR2 | DP_PSR_IRQ_HPD_WITH_CRC_ERRORS;
 409         } else {
 410                 if (dev_priv->psr.link_standby)
 411                         dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
 412
 413                 if (INTEL_GEN(dev_priv) >= 8)
 414                         dpcd_val |= DP_PSR_CRC_VERIFICATION;
 415         }
 416
 417         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
 418
 419         drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
 420 }
 421
 422 static u32 intel_psr1_get_tp_time(struct intel_dp *intel_dp)
 423 {
 424         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 425         u32 val = 0;
 426
 427         if (INTEL_GEN(dev_priv) >= 11)
 428                 val |= EDP_PSR_TP4_TIME_0US;
 429
 430         if (dev_priv->vbt.psr.tp1_wakeup_time_us == 0)
 431                 val |= EDP_PSR_TP1_TIME_0us;
 432         else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 100)
 433                 val |= EDP_PSR_TP1_TIME_100us;
 434         else if (dev_priv->vbt.psr.tp1_wakeup_time_us <= 500)
 435                 val |= EDP_PSR_TP1_TIME_500us;
 436         else
 437                 val |= EDP_PSR_TP1_TIME_2500us;
 438
 439         if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us == 0)
 440                 val |= EDP_PSR_TP2_TP3_TIME_0us;
 441         else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 100)
 442                 val |= EDP_PSR_TP2_TP3_TIME_100us;
 443         else if (dev_priv->vbt.psr.tp2_tp3_wakeup_time_us <= 500)
 444                 val |= EDP_PSR_TP2_TP3_TIME_500us;
 445         else
 446                 val |= EDP_PSR_TP2_TP3_TIME_2500us;
 447
 448         if (intel_dp_source_supports_hbr2(intel_dp) &&
 449             drm_dp_tps3_supported(intel_dp->dpcd))
 450                 val |= EDP_PSR_TP1_TP3_SEL;
 451         else
 452                 val |= EDP_PSR_TP1_TP2_SEL;
 453
 454         return val;
 455 }
 456
 457 static void hsw_activate_psr1(struct intel_dp *intel_dp)
 458 {
 459         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 460         u32 max_sleep_time = 0x1f;
 461         u32 val = EDP_PSR_ENABLE;
 462
 463         /* Let's use 6 as the minimum to cover all known cases including the
 464          * off-by-one issue that HW has in some cases.
 465          */
 466         int idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
 467
 468         /* sink_sync_latency of 8 means source has to wait for more than 8
 469          * frames, we'll go with 9 frames for now
 470          */
 471         idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1);
 472         val |= idle_frames << EDP_PSR_IDLE_FRAME_SHIFT;
 473
 474         val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
 475         if (IS_HASWELL(dev_priv))
 476                 val |= EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
 477
 478         if (dev_priv->psr.link_standby)
 479                 val |= EDP_PSR_LINK_STANDBY;
 480
 481         val |= intel_psr1_get_tp_time(intel_dp);
 482
 483         if (INTEL_GEN(dev_priv) >= 8)
 484                 val |= EDP_PSR_CRC_ENABLE;
 485
 486         val |= (I915_READ(EDP_PSR_CTL(dev_priv->psr.transcoder)) &
 487                 EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK);
 488         I915_WRITE(EDP_PSR_CTL(dev_priv->psr.transcoder), val);
 489 }
 490
 491 static void hsw_activate_psr2(struct intel_dp *intel_dp)
 492 {
 493         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 494         u32 val;
 495
 496         /* Let's use 6 as the minimum to cover all known cases including the
 497          * off-by-one issue that HW has in some cases.
 498          */
 499         int idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
 500
 501         idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1);
 502         val = idle_frames << EDP_PSR2_IDLE_FRAME_SHIFT;
 503
 504         val |= EDP_PSR2_ENABLE | EDP_SU_TRACK_ENABLE;
 505         if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
 506                 val |= EDP_Y_COORDINATE_ENABLE;
 507
 508         val |= EDP_PSR2_FRAME_BEFORE_SU(dev_priv->psr.sink_sync_latency + 1);
 509
 510         if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us >= 0 &&
 511             dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 50)
 512                 val |= EDP_PSR2_TP2_TIME_50us;
 513         else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 100)
 514                 val |= EDP_PSR2_TP2_TIME_100us;
 515         else if (dev_priv->vbt.psr.psr2_tp2_tp3_wakeup_time_us <= 500)
 516                 val |= EDP_PSR2_TP2_TIME_500us;
 517         else
 518                 val |= EDP_PSR2_TP2_TIME_2500us;
 519
 520         /*
 521          * PSR2 HW is incorrectly using EDP_PSR_TP1_TP3_SEL and BSpec is
 522          * recommending keep this bit unset while PSR2 is enabled.
 523          */
 524         I915_WRITE(EDP_PSR_CTL(dev_priv->psr.transcoder), 0);
 525
 526         I915_WRITE(EDP_PSR2_CTL(dev_priv->psr.transcoder), val);
 527 }
 528
 529 static bool
 530 transcoder_has_psr2(struct drm_i915_private *dev_priv, enum transcoder trans)
 531 {
 532         if (INTEL_GEN(dev_priv) < 9)
 533                 return false;
 534         else if (INTEL_GEN(dev_priv) >= 12)
 535                 return trans == TRANSCODER_A;
 536         else
 537                 return trans == TRANSCODER_EDP;
 538 }
 539
 540 static u32 intel_get_frame_time_us(const struct intel_crtc_state *cstate)
 541 {
 542         if (!cstate || !cstate->hw.active)
 543                 return 0;
 544
 545         return DIV_ROUND_UP(1000 * 1000,
 546                             drm_mode_vrefresh(&cstate->hw.adjusted_mode));
 547 }
 548
 549 static void psr2_program_idle_frames(struct drm_i915_private *dev_priv,
 550                                      u32 idle_frames)
 551 {
 552         u32 val;
 553
 554         idle_frames <<=  EDP_PSR2_IDLE_FRAME_SHIFT;
 555         val = I915_READ(EDP_PSR2_CTL(dev_priv->psr.transcoder));
 556         val &= ~EDP_PSR2_IDLE_FRAME_MASK;
 557         val |= idle_frames;
 558         I915_WRITE(EDP_PSR2_CTL(dev_priv->psr.transcoder), val);
 559 }
 560
 561 static void tgl_psr2_enable_dc3co(struct drm_i915_private *dev_priv)
 562 {
 563         psr2_program_idle_frames(dev_priv, 0);
 564         intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_DC3CO);
 565 }
 566
 567 static void tgl_psr2_disable_dc3co(struct drm_i915_private *dev_priv)
 568 {
 569         int idle_frames;
 570
 571         intel_display_power_set_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 572         /*
 573          * Restore PSR2 idle frame let's use 6 as the minimum to cover all known
 574          * cases including the off-by-one issue that HW has in some cases.
 575          */
 576         idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
 577         idle_frames = max(idle_frames, dev_priv->psr.sink_sync_latency + 1);
 578         psr2_program_idle_frames(dev_priv, idle_frames);
 579 }
 580
 581 static void tgl_dc5_idle_thread(struct work_struct *work)
 582 {
 583         struct drm_i915_private *dev_priv =
 584                 container_of(work, typeof(*dev_priv), psr.idle_work.work);
 585
 586         mutex_lock(&dev_priv->psr.lock);
 587         /* If delayed work is pending, it is not idle */
 588         if (delayed_work_pending(&dev_priv->psr.idle_work))
 589                 goto unlock;
 590
 591         DRM_DEBUG_KMS("DC5/6 idle thread\n");
 592         tgl_psr2_disable_dc3co(dev_priv);
 593 unlock:
 594         mutex_unlock(&dev_priv->psr.lock);
 595 }
 596
 597 static void tgl_disallow_dc3co_on_psr2_exit(struct drm_i915_private *dev_priv)
 598 {
 599         if (!dev_priv->psr.dc3co_enabled)
 600                 return;
 601
 602         cancel_delayed_work(&dev_priv->psr.idle_work);
 603         /* Before PSR2 exit disallow dc3co*/
 604         tgl_psr2_disable_dc3co(dev_priv);
 605 }
 606
 607 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
 608                                     struct intel_crtc_state *crtc_state)
 609 {
 610         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 611         int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay;
 612         int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay;
 613         int psr_max_h = 0, psr_max_v = 0, max_bpp = 0;
 614
 615         if (!dev_priv->psr.sink_psr2_support)
 616                 return false;
 617
 618         if (!transcoder_has_psr2(dev_priv, crtc_state->cpu_transcoder)) {
 619                 DRM_DEBUG_KMS("PSR2 not supported in transcoder %s\n",
 620                               transcoder_name(crtc_state->cpu_transcoder));
 621                 return false;
 622         }
 623
 624         /*
 625          * DSC and PSR2 cannot be enabled simultaneously. If a requested
 626          * resolution requires DSC to be enabled, priority is given to DSC
 627          * over PSR2.
 628          */
 629         if (crtc_state->dsc.compression_enable) {
 630                 DRM_DEBUG_KMS("PSR2 cannot be enabled since DSC is enabled\n");
 631                 return false;
 632         }
 633
 634         if (INTEL_GEN(dev_priv) >= 12) {
 635                 psr_max_h = 5120;
 636                 psr_max_v = 3200;
 637                 max_bpp = 30;
 638         } else if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
 639                 psr_max_h = 4096;
 640                 psr_max_v = 2304;
 641                 max_bpp = 24;
 642         } else if (IS_GEN(dev_priv, 9)) {
 643                 psr_max_h = 3640;
 644                 psr_max_v = 2304;
 645                 max_bpp = 24;
 646         }
 647
 648         if (crtc_hdisplay > psr_max_h || crtc_vdisplay > psr_max_v) {
 649                 DRM_DEBUG_KMS("PSR2 not enabled, resolution %dx%d > max supported %dx%d\n",
 650                               crtc_hdisplay, crtc_vdisplay,
 651                               psr_max_h, psr_max_v);
 652                 return false;
 653         }
 654
 655         if (crtc_state->pipe_bpp > max_bpp) {
 656                 DRM_DEBUG_KMS("PSR2 not enabled, pipe bpp %d > max supported %d\n",
 657                               crtc_state->pipe_bpp, max_bpp);
 658                 return false;
 659         }
 660
 661         /*
 662          * HW sends SU blocks of size four scan lines, which means the starting
 663          * X coordinate and Y granularity requirements will always be met. We
 664          * only need to validate the SU block width is a multiple of
 665          * x granularity.
 666          */
 667         if (crtc_hdisplay % dev_priv->psr.su_x_granularity) {
 668                 DRM_DEBUG_KMS("PSR2 not enabled, hdisplay(%d) not multiple of %d\n",
 669                               crtc_hdisplay, dev_priv->psr.su_x_granularity);
 670                 return false;
 671         }
 672
 673         if (crtc_state->crc_enabled) {
 674                 DRM_DEBUG_KMS("PSR2 not enabled because it would inhibit pipe CRC calculation\n");
 675                 return false;
 676         }
 677
 678         return true;
 679 }
 680
 681 void intel_psr_compute_config(struct intel_dp *intel_dp,
 682                               struct intel_crtc_state *crtc_state)
 683 {
 684         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 685         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 686         const struct drm_display_mode *adjusted_mode =
 687                 &crtc_state->hw.adjusted_mode;
 688         int psr_setup_time;
 689
 690         if (!CAN_PSR(dev_priv))
 691                 return;
 692
 693         if (intel_dp != dev_priv->psr.dp)
 694                 return;
 695
 696         /*
 697          * HSW spec explicitly says PSR is tied to port A.
 698          * BDW+ platforms have a instance of PSR registers per transcoder but
 699          * for now it only supports one instance of PSR, so lets keep it
 700          * hardcoded to PORT_A
 701          */
 702         if (dig_port->base.port != PORT_A) {
 703                 DRM_DEBUG_KMS("PSR condition failed: Port not supported\n");
 704                 return;
 705         }
 706
 707         if (dev_priv->psr.sink_not_reliable) {
 708                 DRM_DEBUG_KMS("PSR sink implementation is not reliable\n");
 709                 return;
 710         }
 711
 712         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
 713                 DRM_DEBUG_KMS("PSR condition failed: Interlaced mode enabled\n");
 714                 return;
 715         }
 716
 717         psr_setup_time = drm_dp_psr_setup_time(intel_dp->psr_dpcd);
 718         if (psr_setup_time < 0) {
 719                 DRM_DEBUG_KMS("PSR condition failed: Invalid PSR setup time (0x%02x)\n",
 720                               intel_dp->psr_dpcd[1]);
 721                 return;
 722         }
 723
 724         if (intel_usecs_to_scanlines(adjusted_mode, psr_setup_time) >
 725             adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vdisplay - 1) {
 726                 DRM_DEBUG_KMS("PSR condition failed: PSR setup time (%d us) too long\n",
 727                               psr_setup_time);
 728                 return;
 729         }
 730
 731         crtc_state->has_psr = true;
 732         crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state);
 733 }
 734
 735 static void intel_psr_activate(struct intel_dp *intel_dp)
 736 {
 737         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 738
 739         if (transcoder_has_psr2(dev_priv, dev_priv->psr.transcoder))
 740                 WARN_ON(I915_READ(EDP_PSR2_CTL(dev_priv->psr.transcoder)) & EDP_PSR2_ENABLE);
 741
 742         WARN_ON(I915_READ(EDP_PSR_CTL(dev_priv->psr.transcoder)) & EDP_PSR_ENABLE);
 743         WARN_ON(dev_priv->psr.active);
 744         lockdep_assert_held(&dev_priv->psr.lock);
 745
 746         /* psr1 and psr2 are mutually exclusive.*/
 747         if (dev_priv->psr.psr2_enabled)
 748                 hsw_activate_psr2(intel_dp);
 749         else
 750                 hsw_activate_psr1(intel_dp);
 751
 752         dev_priv->psr.active = true;
 753 }
 754
 755 static void intel_psr_enable_source(struct intel_dp *intel_dp,
 756                                     const struct intel_crtc_state *crtc_state)
 757 {
 758         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 759         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 760         u32 mask;
 761
 762         /* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+
 763          * use hardcoded values PSR AUX transactions
 764          */
 765         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 766                 hsw_psr_setup_aux(intel_dp);
 767
 768         if (dev_priv->psr.psr2_enabled && (IS_GEN(dev_priv, 9) &&
 769                                            !IS_GEMINILAKE(dev_priv))) {
 770                 i915_reg_t reg = CHICKEN_TRANS(cpu_transcoder);
 771                 u32 chicken = I915_READ(reg);
 772
 773                 chicken |= PSR2_VSC_ENABLE_PROG_HEADER |
 774                            PSR2_ADD_VERTICAL_LINE_COUNT;
 775                 I915_WRITE(reg, chicken);
 776         }
 777
 778         /*
 779          * Per Spec: Avoid continuous PSR exit by masking MEMUP and HPD also
 780          * mask LPSP to avoid dependency on other drivers that might block
 781          * runtime_pm besides preventing  other hw tracking issues now we
 782          * can rely on frontbuffer tracking.
 783          */
 784         mask = EDP_PSR_DEBUG_MASK_MEMUP |
 785                EDP_PSR_DEBUG_MASK_HPD |
 786                EDP_PSR_DEBUG_MASK_LPSP |
 787                EDP_PSR_DEBUG_MASK_MAX_SLEEP;
 788
 789         if (INTEL_GEN(dev_priv) < 11)
 790                 mask |= EDP_PSR_DEBUG_MASK_DISP_REG_WRITE;
 791
 792         I915_WRITE(EDP_PSR_DEBUG(dev_priv->psr.transcoder), mask);
 793
 794         psr_irq_control(dev_priv);
 795 }
 796
 797 static void intel_psr_enable_locked(struct drm_i915_private *dev_priv,
 798                                     const struct intel_crtc_state *crtc_state)
 799 {
 800         struct intel_dp *intel_dp = dev_priv->psr.dp;
 801         u32 val;
 802
 803         WARN_ON(dev_priv->psr.enabled);
 804
 805         dev_priv->psr.psr2_enabled = intel_psr2_enabled(dev_priv, crtc_state);
 806         dev_priv->psr.busy_frontbuffer_bits = 0;
 807         dev_priv->psr.pipe = to_intel_crtc(crtc_state->uapi.crtc)->pipe;
 808         dev_priv->psr.dc3co_enabled = !!crtc_state->dc3co_exitline;
 809         dev_priv->psr.dc3co_exit_delay = intel_get_frame_time_us(crtc_state);
 810         dev_priv->psr.transcoder = crtc_state->cpu_transcoder;
 811
 812         /*
 813          * If a PSR error happened and the driver is reloaded, the EDP_PSR_IIR
 814          * will still keep the error set even after the reset done in the
 815          * irq_preinstall and irq_uninstall hooks.
 816          * And enabling in this situation cause the screen to freeze in the
 817          * first time that PSR HW tries to activate so lets keep PSR disabled
 818          * to avoid any rendering problems.
 819          */
 820         if (INTEL_GEN(dev_priv) >= 12) {
 821                 val = I915_READ(TRANS_PSR_IIR(dev_priv->psr.transcoder));
 822                 val &= EDP_PSR_ERROR(0);
 823         } else {
 824                 val = I915_READ(EDP_PSR_IIR);
 825                 val &= EDP_PSR_ERROR(dev_priv->psr.transcoder);
 826         }
 827         if (val) {
 828                 dev_priv->psr.sink_not_reliable = true;
 829                 DRM_DEBUG_KMS("PSR interruption error set, not enabling PSR\n");
 830                 return;
 831         }
 832
 833         DRM_DEBUG_KMS("Enabling PSR%s\n",
 834                       dev_priv->psr.psr2_enabled ? "2" : "1");
 835         intel_psr_setup_vsc(intel_dp, crtc_state);
 836         intel_psr_enable_sink(intel_dp);
 837         intel_psr_enable_source(intel_dp, crtc_state);
 838         dev_priv->psr.enabled = true;
 839
 840         intel_psr_activate(intel_dp);
 841 }
 842
 843 /**
 844  * intel_psr_enable - Enable PSR
 845  * @intel_dp: Intel DP
 846  * @crtc_state: new CRTC state
 847  *
 848  * This function can only be called after the pipe is fully trained and enabled.
 849  */
 850 void intel_psr_enable(struct intel_dp *intel_dp,
 851                       const struct intel_crtc_state *crtc_state)
 852 {
 853         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 854
 855         if (!CAN_PSR(dev_priv) || dev_priv->psr.dp != intel_dp)
 856                 return;
 857
 858         dev_priv->psr.force_mode_changed = false;
 859
 860         if (!crtc_state->has_psr)
 861                 return;
 862
 863         WARN_ON(dev_priv->drrs.dp);
 864
 865         mutex_lock(&dev_priv->psr.lock);
 866
 867         if (!psr_global_enabled(dev_priv->psr.debug)) {
 868                 DRM_DEBUG_KMS("PSR disabled by flag\n");
 869                 goto unlock;
 870         }
 871
 872         intel_psr_enable_locked(dev_priv, crtc_state);
 873
 874 unlock:
 875         mutex_unlock(&dev_priv->psr.lock);
 876 }
 877
 878 static void intel_psr_exit(struct drm_i915_private *dev_priv)
 879 {
 880         u32 val;
 881
 882         if (!dev_priv->psr.active) {
 883                 if (transcoder_has_psr2(dev_priv, dev_priv->psr.transcoder)) {
 884                         val = I915_READ(EDP_PSR2_CTL(dev_priv->psr.transcoder));
 885                         WARN_ON(val & EDP_PSR2_ENABLE);
 886                 }
 887
 888                 val = I915_READ(EDP_PSR_CTL(dev_priv->psr.transcoder));
 889                 WARN_ON(val & EDP_PSR_ENABLE);
 890
 891                 return;
 892         }
 893
 894         if (dev_priv->psr.psr2_enabled) {
 895                 tgl_disallow_dc3co_on_psr2_exit(dev_priv);
 896                 val = I915_READ(EDP_PSR2_CTL(dev_priv->psr.transcoder));
 897                 WARN_ON(!(val & EDP_PSR2_ENABLE));
 898                 val &= ~EDP_PSR2_ENABLE;
 899                 I915_WRITE(EDP_PSR2_CTL(dev_priv->psr.transcoder), val);
 900         } else {
 901                 val = I915_READ(EDP_PSR_CTL(dev_priv->psr.transcoder));
 902                 WARN_ON(!(val & EDP_PSR_ENABLE));
 903                 val &= ~EDP_PSR_ENABLE;
 904                 I915_WRITE(EDP_PSR_CTL(dev_priv->psr.transcoder), val);
 905         }
 906         dev_priv->psr.active = false;
 907 }
 908
 909 static void intel_psr_disable_locked(struct intel_dp *intel_dp)
 910 {
 911         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 912         i915_reg_t psr_status;
 913         u32 psr_status_mask;
 914
 915         lockdep_assert_held(&dev_priv->psr.lock);
 916
 917         if (!dev_priv->psr.enabled)
 918                 return;
 919
 920         DRM_DEBUG_KMS("Disabling PSR%s\n",
 921                       dev_priv->psr.psr2_enabled ? "2" : "1");
 922
 923         intel_psr_exit(dev_priv);
 924
 925         if (dev_priv->psr.psr2_enabled) {
 926                 psr_status = EDP_PSR2_STATUS(dev_priv->psr.transcoder);
 927                 psr_status_mask = EDP_PSR2_STATUS_STATE_MASK;
 928         } else {
 929                 psr_status = EDP_PSR_STATUS(dev_priv->psr.transcoder);
 930                 psr_status_mask = EDP_PSR_STATUS_STATE_MASK;
 931         }
 932
 933         /* Wait till PSR is idle */
 934         if (intel_de_wait_for_clear(dev_priv, psr_status,
 935                                     psr_status_mask, 2000))
 936                 DRM_ERROR("Timed out waiting PSR idle state\n");
 937
 938         /* Disable PSR on Sink */
 939         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
 940
 941         if (dev_priv->psr.psr2_enabled)
 942                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_RECEIVER_ALPM_CONFIG, 0);
 943
 944         dev_priv->psr.enabled = false;
 945 }
 946
 947 /**
 948  * intel_psr_disable - Disable PSR
 949  * @intel_dp: Intel DP
 950  * @old_crtc_state: old CRTC state
 951  *
 952  * This function needs to be called before disabling pipe.
 953  */
 954 void intel_psr_disable(struct intel_dp *intel_dp,
 955                        const struct intel_crtc_state *old_crtc_state)
 956 {
 957         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 958
 959         if (!old_crtc_state->has_psr)
 960                 return;
 961
 962         if (WARN_ON(!CAN_PSR(dev_priv)))
 963                 return;
 964
 965         mutex_lock(&dev_priv->psr.lock);
 966
 967         intel_psr_disable_locked(intel_dp);
 968
 969         mutex_unlock(&dev_priv->psr.lock);
 970         cancel_work_sync(&dev_priv->psr.work);
 971         cancel_delayed_work_sync(&dev_priv->psr.idle_work);
 972 }
 973
 974 static void psr_force_hw_tracking_exit(struct drm_i915_private *dev_priv)
 975 {
 976         if (INTEL_GEN(dev_priv) >= 9)
 977                 /*
 978                  * Display WA #0884: skl+
 979                  * This documented WA for bxt can be safely applied
 980                  * broadly so we can force HW tracking to exit PSR
 981                  * instead of disabling and re-enabling.
 982                  * Workaround tells us to write 0 to CUR_SURFLIVE_A,
 983                  * but it makes more sense write to the current active
 984                  * pipe.
 985                  */
 986                 I915_WRITE(CURSURFLIVE(dev_priv->psr.pipe), 0);
 987         else
 988                 /*
 989                  * A write to CURSURFLIVE do not cause HW tracking to exit PSR
 990                  * on older gens so doing the manual exit instead.
 991                  */
 992                 intel_psr_exit(dev_priv);
 993 }
 994
 995 /**
 996  * intel_psr_update - Update PSR state
 997  * @intel_dp: Intel DP
 998  * @crtc_state: new CRTC state
 999  *
1000  * This functions will update PSR states, disabling, enabling or switching PSR
1001  * version when executing fastsets. For full modeset, intel_psr_disable() and
1002  * intel_psr_enable() should be called instead.
1003  */
1004 void intel_psr_update(struct intel_dp *intel_dp,
1005                       const struct intel_crtc_state *crtc_state)
1006 {
1007         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1008         struct i915_psr *psr = &dev_priv->psr;
1009         bool enable, psr2_enable;
1010
1011         if (!CAN_PSR(dev_priv) || READ_ONCE(psr->dp) != intel_dp)
1012                 return;
1013
1014         dev_priv->psr.force_mode_changed = false;
1015
1016         mutex_lock(&dev_priv->psr.lock);
1017
1018         enable = crtc_state->has_psr && psr_global_enabled(psr->debug);
1019         psr2_enable = intel_psr2_enabled(dev_priv, crtc_state);
1020
1021         if (enable == psr->enabled && psr2_enable == psr->psr2_enabled) {
1022                 /* Force a PSR exit when enabling CRC to avoid CRC timeouts */
1023                 if (crtc_state->crc_enabled && psr->enabled)
1024                         psr_force_hw_tracking_exit(dev_priv);
1025                 else if (INTEL_GEN(dev_priv) < 9 && psr->enabled) {
1026                         /*
1027                          * Activate PSR again after a force exit when enabling
1028                          * CRC in older gens
1029                          */
1030                         if (!dev_priv->psr.active &&
1031                             !dev_priv->psr.busy_frontbuffer_bits)
1032                                 schedule_work(&dev_priv->psr.work);
1033                 }
1034
1035                 goto unlock;
1036         }
1037
1038         if (psr->enabled)
1039                 intel_psr_disable_locked(intel_dp);
1040
1041         if (enable)
1042                 intel_psr_enable_locked(dev_priv, crtc_state);
1043
1044 unlock:
1045         mutex_unlock(&dev_priv->psr.lock);
1046 }
1047
1048 /**
1049  * intel_psr_wait_for_idle - wait for PSR1 to idle
1050  * @new_crtc_state: new CRTC state
1051  * @out_value: PSR status in case of failure
1052  *
1053  * This function is expected to be called from pipe_update_start() where it is
1054  * not expected to race with PSR enable or disable.
1055  *
1056  * Returns: 0 on success or -ETIMEOUT if PSR status does not idle.
1057  */
1058 int intel_psr_wait_for_idle(const struct intel_crtc_state *new_crtc_state,
1059                             u32 *out_value)
1060 {
1061         struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
1062         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1063
1064         if (!dev_priv->psr.enabled || !new_crtc_state->has_psr)
1065                 return 0;
1066
1067         /* FIXME: Update this for PSR2 if we need to wait for idle */
1068         if (READ_ONCE(dev_priv->psr.psr2_enabled))
1069                 return 0;
1070
1071         /*
1072          * From bspec: Panel Self Refresh (BDW+)
1073          * Max. time for PSR to idle = Inverse of the refresh rate + 6 ms of
1074          * exit training time + 1.5 ms of aux channel handshake. 50 ms is
1075          * defensive enough to cover everything.
1076          */
1077
1078         return __intel_wait_for_register(&dev_priv->uncore,
1079                                          EDP_PSR_STATUS(dev_priv->psr.transcoder),
1080                                          EDP_PSR_STATUS_STATE_MASK,
1081                                          EDP_PSR_STATUS_STATE_IDLE, 2, 50,
1082                                          out_value);
1083 }
1084
1085 static bool __psr_wait_for_idle_locked(struct drm_i915_private *dev_priv)
1086 {
1087         i915_reg_t reg;
1088         u32 mask;
1089         int err;
1090
1091         if (!dev_priv->psr.enabled)
1092                 return false;
1093
1094         if (dev_priv->psr.psr2_enabled) {
1095                 reg = EDP_PSR2_STATUS(dev_priv->psr.transcoder);
1096                 mask = EDP_PSR2_STATUS_STATE_MASK;
1097         } else {
1098                 reg = EDP_PSR_STATUS(dev_priv->psr.transcoder);
1099                 mask = EDP_PSR_STATUS_STATE_MASK;
1100         }
1101
1102         mutex_unlock(&dev_priv->psr.lock);
1103
1104         err = intel_de_wait_for_clear(dev_priv, reg, mask, 50);
1105         if (err)
1106                 DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
1107
1108         /* After the unlocked wait, verify that PSR is still wanted! */
1109         mutex_lock(&dev_priv->psr.lock);
1110         return err == 0 && dev_priv->psr.enabled;
1111 }
1112
1113 static int intel_psr_fastset_force(struct drm_i915_private *dev_priv)
1114 {
1115         struct drm_device *dev = &dev_priv->drm;
1116         struct drm_modeset_acquire_ctx ctx;
1117         struct drm_atomic_state *state;
1118         struct intel_crtc *crtc;
1119         int err;
1120
1121         state = drm_atomic_state_alloc(dev);
1122         if (!state)
1123                 return -ENOMEM;
1124
1125         drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
1126         state->acquire_ctx = &ctx;
1127
1128 retry:
1129         for_each_intel_crtc(dev, crtc) {
1130                 struct intel_crtc_state *crtc_state =
1131                         intel_atomic_get_crtc_state(state, crtc);
1132
1133                 if (IS_ERR(crtc_state)) {
1134                         err = PTR_ERR(crtc_state);
1135                         goto error;
1136                 }
1137
1138                 if (crtc_state->hw.active && crtc_state->has_psr) {
1139                         /* Mark mode as changed to trigger a pipe->update() */
1140                         crtc_state->uapi.mode_changed = true;
1141                         break;
1142                 }
1143         }
1144
1145         err = drm_atomic_commit(state);
1146
1147 error:
1148         if (err == -EDEADLK) {
1149                 drm_atomic_state_clear(state);
1150                 err = drm_modeset_backoff(&ctx);
1151                 if (!err)
1152                         goto retry;
1153         }
1154
1155         drm_modeset_drop_locks(&ctx);
1156         drm_modeset_acquire_fini(&ctx);
1157         drm_atomic_state_put(state);
1158
1159         return err;
1160 }
1161
1162 int intel_psr_debug_set(struct drm_i915_private *dev_priv, u64 val)
1163 {
1164         const u32 mode = val & I915_PSR_DEBUG_MODE_MASK;
1165         u32 old_mode;
1166         int ret;
1167
1168         if (val & ~(I915_PSR_DEBUG_IRQ | I915_PSR_DEBUG_MODE_MASK) ||
1169             mode > I915_PSR_DEBUG_FORCE_PSR1) {
1170                 DRM_DEBUG_KMS("Invalid debug mask %llx\n", val);
1171                 return -EINVAL;
1172         }
1173
1174         ret = mutex_lock_interruptible(&dev_priv->psr.lock);
1175         if (ret)
1176                 return ret;
1177
1178         old_mode = dev_priv->psr.debug & I915_PSR_DEBUG_MODE_MASK;
1179         dev_priv->psr.debug = val;
1180
1181         /*
1182          * Do it right away if it's already enabled, otherwise it will be done
1183          * when enabling the source.
1184          */
1185         if (dev_priv->psr.enabled)
1186                 psr_irq_control(dev_priv);
1187
1188         mutex_unlock(&dev_priv->psr.lock);
1189
1190         if (old_mode != mode)
1191                 ret = intel_psr_fastset_force(dev_priv);
1192
1193         return ret;
1194 }
1195
1196 static void intel_psr_handle_irq(struct drm_i915_private *dev_priv)
1197 {
1198         struct i915_psr *psr = &dev_priv->psr;
1199
1200         intel_psr_disable_locked(psr->dp);
1201         psr->sink_not_reliable = true;
1202         /* let's make sure that sink is awaken */
1203         drm_dp_dpcd_writeb(&psr->dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
1204 }
1205
1206 static void intel_psr_work(struct work_struct *work)
1207 {
1208         struct drm_i915_private *dev_priv =
1209                 container_of(work, typeof(*dev_priv), psr.work);
1210
1211         mutex_lock(&dev_priv->psr.lock);
1212
1213         if (!dev_priv->psr.enabled)
1214                 goto unlock;
1215
1216         if (READ_ONCE(dev_priv->psr.irq_aux_error))
1217                 intel_psr_handle_irq(dev_priv);
1218
1219         /*
1220          * We have to make sure PSR is ready for re-enable
1221          * otherwise it keeps disabled until next full enable/disable cycle.
1222          * PSR might take some time to get fully disabled
1223          * and be ready for re-enable.
1224          */
1225         if (!__psr_wait_for_idle_locked(dev_priv))
1226                 goto unlock;
1227
1228         /*
1229          * The delayed work can race with an invalidate hence we need to
1230          * recheck. Since psr_flush first clears this and then reschedules we
1231          * won't ever miss a flush when bailing out here.
1232          */
1233         if (dev_priv->psr.busy_frontbuffer_bits || dev_priv->psr.active)
1234                 goto unlock;
1235
1236         intel_psr_activate(dev_priv->psr.dp);
1237 unlock:
1238         mutex_unlock(&dev_priv->psr.lock);
1239 }
1240
1241 /**
1242  * intel_psr_invalidate - Invalidade PSR
1243  * @dev_priv: i915 device
1244  * @frontbuffer_bits: frontbuffer plane tracking bits
1245  * @origin: which operation caused the invalidate
1246  *
1247  * Since the hardware frontbuffer tracking has gaps we need to integrate
1248  * with the software frontbuffer tracking. This function gets called every
1249  * time frontbuffer rendering starts and a buffer gets dirtied. PSR must be
1250  * disabled if the frontbuffer mask contains a buffer relevant to PSR.
1251  *
1252  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits."
1253  */
1254 void intel_psr_invalidate(struct drm_i915_private *dev_priv,
1255                           unsigned frontbuffer_bits, enum fb_op_origin origin)
1256 {
1257         if (!CAN_PSR(dev_priv))
1258                 return;
1259
1260         if (origin == ORIGIN_FLIP)
1261                 return;
1262
1263         mutex_lock(&dev_priv->psr.lock);
1264         if (!dev_priv->psr.enabled) {
1265                 mutex_unlock(&dev_priv->psr.lock);
1266                 return;
1267         }
1268
1269         frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(dev_priv->psr.pipe);
1270         dev_priv->psr.busy_frontbuffer_bits |= frontbuffer_bits;
1271
1272         if (frontbuffer_bits)
1273                 intel_psr_exit(dev_priv);
1274
1275         mutex_unlock(&dev_priv->psr.lock);
1276 }
1277
1278 /*
1279  * When we will be completely rely on PSR2 S/W tracking in future,
1280  * intel_psr_flush() will invalidate and flush the PSR for ORIGIN_FLIP
1281  * event also therefore tgl_dc3co_flush() require to be changed
1282  * accrodingly in future.
1283  */
1284 static void
1285 tgl_dc3co_flush(struct drm_i915_private *dev_priv,
1286                 unsigned int frontbuffer_bits, enum fb_op_origin origin)
1287 {
1288         u32 delay;
1289
1290         mutex_lock(&dev_priv->psr.lock);
1291
1292         if (!dev_priv->psr.dc3co_enabled)
1293                 goto unlock;
1294
1295         if (!dev_priv->psr.psr2_enabled || !dev_priv->psr.active)
1296                 goto unlock;
1297
1298         /*
1299          * At every frontbuffer flush flip event modified delay of delayed work,
1300          * when delayed work schedules that means display has been idle.
1301          */
1302         if (!(frontbuffer_bits &
1303             INTEL_FRONTBUFFER_ALL_MASK(dev_priv->psr.pipe)))
1304                 goto unlock;
1305
1306         tgl_psr2_enable_dc3co(dev_priv);
1307         /* DC5/DC6 required idle frames = 6 */
1308         delay = 6 * dev_priv->psr.dc3co_exit_delay;
1309         mod_delayed_work(system_wq, &dev_priv->psr.idle_work,
1310                          usecs_to_jiffies(delay));
1311
1312 unlock:
1313         mutex_unlock(&dev_priv->psr.lock);
1314 }
1315
1316 /**
1317  * intel_psr_flush - Flush PSR
1318  * @dev_priv: i915 device
1319  * @frontbuffer_bits: frontbuffer plane tracking bits
1320  * @origin: which operation caused the flush
1321  *
1322  * Since the hardware frontbuffer tracking has gaps we need to integrate
1323  * with the software frontbuffer tracking. This function gets called every
1324  * time frontbuffer rendering has completed and flushed out to memory. PSR
1325  * can be enabled again if no other frontbuffer relevant to PSR is dirty.
1326  *
1327  * Dirty frontbuffers relevant to PSR are tracked in busy_frontbuffer_bits.
1328  */
1329 void intel_psr_flush(struct drm_i915_private *dev_priv,
1330                      unsigned frontbuffer_bits, enum fb_op_origin origin)
1331 {
1332         if (!CAN_PSR(dev_priv))
1333                 return;
1334
1335         if (origin == ORIGIN_FLIP) {
1336                 tgl_dc3co_flush(dev_priv, frontbuffer_bits, origin);
1337                 return;
1338         }
1339
1340         mutex_lock(&dev_priv->psr.lock);
1341         if (!dev_priv->psr.enabled) {
1342                 mutex_unlock(&dev_priv->psr.lock);
1343                 return;
1344         }
1345
1346         frontbuffer_bits &= INTEL_FRONTBUFFER_ALL_MASK(dev_priv->psr.pipe);
1347         dev_priv->psr.busy_frontbuffer_bits &= ~frontbuffer_bits;
1348
1349         /* By definition flush = invalidate + flush */
1350         if (frontbuffer_bits)
1351                 psr_force_hw_tracking_exit(dev_priv);
1352
1353         if (!dev_priv->psr.active && !dev_priv->psr.busy_frontbuffer_bits)
1354                 schedule_work(&dev_priv->psr.work);
1355         mutex_unlock(&dev_priv->psr.lock);
1356 }
1357
1358 /**
1359  * intel_psr_init - Init basic PSR work and mutex.
1360  * @dev_priv: i915 device private
1361  *
1362  * This function is  called only once at driver load to initialize basic
1363  * PSR stuff.
1364  */
1365 void intel_psr_init(struct drm_i915_private *dev_priv)
1366 {
1367         if (!HAS_PSR(dev_priv))
1368                 return;
1369
1370         if (!dev_priv->psr.sink_support)
1371                 return;
1372
1373         if (IS_HASWELL(dev_priv))
1374                 /*
1375                  * HSW don't have PSR registers on the same space as transcoder
1376                  * so set this to a value that when subtract to the register
1377                  * in transcoder space results in the right offset for HSW
1378                  */
1379                 dev_priv->hsw_psr_mmio_adjust = _SRD_CTL_EDP - _HSW_EDP_PSR_BASE;
1380
1381         if (i915_modparams.enable_psr == -1)
1382                 if (INTEL_GEN(dev_priv) < 9 || !dev_priv->vbt.psr.enable)
1383                         i915_modparams.enable_psr = 0;
1384
1385         /* Set link_standby x link_off defaults */
1386         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
1387                 /* HSW and BDW require workarounds that we don't implement. */
1388                 dev_priv->psr.link_standby = false;
1389         else if (INTEL_GEN(dev_priv) < 12)
1390                 /* For new platforms up to TGL let's respect VBT back again */
1391                 dev_priv->psr.link_standby = dev_priv->vbt.psr.full_link;
1392
1393         INIT_WORK(&dev_priv->psr.work, intel_psr_work);
1394         INIT_DELAYED_WORK(&dev_priv->psr.idle_work, tgl_dc5_idle_thread);
1395         mutex_init(&dev_priv->psr.lock);
1396 }
1397
1398 static int psr_get_status_and_error_status(struct intel_dp *intel_dp,
1399                                            u8 *status, u8 *error_status)
1400 {
1401         struct drm_dp_aux *aux = &intel_dp->aux;
1402         int ret;
1403
1404         ret = drm_dp_dpcd_readb(aux, DP_PSR_STATUS, status);
1405         if (ret != 1)
1406                 return ret;
1407
1408         ret = drm_dp_dpcd_readb(aux, DP_PSR_ERROR_STATUS, error_status);
1409         if (ret != 1)
1410                 return ret;
1411
1412         *status = *status & DP_PSR_SINK_STATE_MASK;
1413
1414         return 0;
1415 }
1416
1417 static void psr_alpm_check(struct intel_dp *intel_dp)
1418 {
1419         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1420         struct drm_dp_aux *aux = &intel_dp->aux;
1421         struct i915_psr *psr = &dev_priv->psr;
1422         u8 val;
1423         int r;
1424
1425         if (!psr->psr2_enabled)
1426                 return;
1427
1428         r = drm_dp_dpcd_readb(aux, DP_RECEIVER_ALPM_STATUS, &val);
1429         if (r != 1) {
1430                 DRM_ERROR("Error reading ALPM status\n");
1431                 return;
1432         }
1433
1434         if (val & DP_ALPM_LOCK_TIMEOUT_ERROR) {
1435                 intel_psr_disable_locked(intel_dp);
1436                 psr->sink_not_reliable = true;
1437                 DRM_DEBUG_KMS("ALPM lock timeout error, disabling PSR\n");
1438
1439                 /* Clearing error */
1440                 drm_dp_dpcd_writeb(aux, DP_RECEIVER_ALPM_STATUS, val);
1441         }
1442 }
1443
1444 static void psr_capability_changed_check(struct intel_dp *intel_dp)
1445 {
1446         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1447         struct i915_psr *psr = &dev_priv->psr;
1448         u8 val;
1449         int r;
1450
1451         r = drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ESI, &val);
1452         if (r != 1) {
1453                 DRM_ERROR("Error reading DP_PSR_ESI\n");
1454                 return;
1455         }
1456
1457         if (val & DP_PSR_CAPS_CHANGE) {
1458                 intel_psr_disable_locked(intel_dp);
1459                 psr->sink_not_reliable = true;
1460                 DRM_DEBUG_KMS("Sink PSR capability changed, disabling PSR\n");
1461
1462                 /* Clearing it */
1463                 drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ESI, val);
1464         }
1465 }
1466
1467 void intel_psr_short_pulse(struct intel_dp *intel_dp)
1468 {
1469         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1470         struct i915_psr *psr = &dev_priv->psr;
1471         u8 status, error_status;
1472         const u8 errors = DP_PSR_RFB_STORAGE_ERROR |
1473                           DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
1474                           DP_PSR_LINK_CRC_ERROR;
1475
1476         if (!CAN_PSR(dev_priv) || !intel_dp_is_edp(intel_dp))
1477                 return;
1478
1479         mutex_lock(&psr->lock);
1480
1481         if (!psr->enabled || psr->dp != intel_dp)
1482                 goto exit;
1483
1484         if (psr_get_status_and_error_status(intel_dp, &status, &error_status)) {
1485                 DRM_ERROR("Error reading PSR status or error status\n");
1486                 goto exit;
1487         }
1488
1489         if (status == DP_PSR_SINK_INTERNAL_ERROR || (error_status & errors)) {
1490                 intel_psr_disable_locked(intel_dp);
1491                 psr->sink_not_reliable = true;
1492         }
1493
1494         if (status == DP_PSR_SINK_INTERNAL_ERROR && !error_status)
1495                 DRM_DEBUG_KMS("PSR sink internal error, disabling PSR\n");
1496         if (error_status & DP_PSR_RFB_STORAGE_ERROR)
1497                 DRM_DEBUG_KMS("PSR RFB storage error, disabling PSR\n");
1498         if (error_status & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
1499                 DRM_DEBUG_KMS("PSR VSC SDP uncorrectable error, disabling PSR\n");
1500         if (error_status & DP_PSR_LINK_CRC_ERROR)
1501                 DRM_DEBUG_KMS("PSR Link CRC error, disabling PSR\n");
1502
1503         if (error_status & ~errors)
1504                 DRM_ERROR("PSR_ERROR_STATUS unhandled errors %x\n",
1505                           error_status & ~errors);
1506         /* clear status register */
1507         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, error_status);
1508
1509         psr_alpm_check(intel_dp);
1510         psr_capability_changed_check(intel_dp);
1511
1512 exit:
1513         mutex_unlock(&psr->lock);
1514 }
1515
1516 bool intel_psr_enabled(struct intel_dp *intel_dp)
1517 {
1518         struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
1519         bool ret;
1520
1521         if (!CAN_PSR(dev_priv) || !intel_dp_is_edp(intel_dp))
1522                 return false;
1523
1524         mutex_lock(&dev_priv->psr.lock);
1525         ret = (dev_priv->psr.dp == intel_dp && dev_priv->psr.enabled);
1526         mutex_unlock(&dev_priv->psr.lock);
1527
1528         return ret;
1529 }
1530
1531 void intel_psr_atomic_check(struct drm_connector *connector,
1532                             struct drm_connector_state *old_state,
1533                             struct drm_connector_state *new_state)
1534 {
1535         struct drm_i915_private *dev_priv = to_i915(connector->dev);
1536         struct intel_connector *intel_connector;
1537         struct intel_digital_port *dig_port;
1538         struct drm_crtc_state *crtc_state;
1539
1540         if (!CAN_PSR(dev_priv) || !new_state->crtc ||
1541             !dev_priv->psr.force_mode_changed)
1542                 return;
1543
1544         intel_connector = to_intel_connector(connector);
1545         dig_port = enc_to_dig_port(intel_connector->encoder);
1546         if (dev_priv->psr.dp != &dig_port->dp)
1547                 return;
1548
1549         crtc_state = drm_atomic_get_new_crtc_state(new_state->state,
1550                                                    new_state->crtc);
1551         crtc_state->mode_changed = true;
1552 }
1553
1554 void intel_psr_set_force_mode_changed(struct intel_dp *intel_dp)
1555 {
1556         struct drm_i915_private *dev_priv;
1557
1558         if (!intel_dp)
1559                 return;
1560
1561         dev_priv = dp_to_i915(intel_dp);
1562         if (!CAN_PSR(dev_priv) || intel_dp != dev_priv->psr.dp)
1563                 return;
1564
1565         dev_priv->psr.force_mode_changed = true;
1566 }