drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/time.h>
  25
  26 #include "hsw_ips.h"
  27 #include "i915_reg.h"
  28 #include "intel_atomic.h"
  29 #include "intel_atomic_plane.h"
  30 #include "intel_audio.h"
  31 #include "intel_bw.h"
  32 #include "intel_cdclk.h"
  33 #include "intel_crtc.h"
  34 #include "intel_de.h"
  35 #include "intel_display_types.h"
  36 #include "intel_mchbar_regs.h"
  37 #include "intel_pci_config.h"
  38 #include "intel_pcode.h"
  39 #include "intel_psr.h"
  40 #include "vlv_sideband.h"
  41
  42 /**
  43  * DOC: CDCLK / RAWCLK
  44  *
  45  * The display engine uses several different clocks to do its work. There
  46  * are two main clocks involved that aren't directly related to the actual
  47  * pixel clock or any symbol/bit clock of the actual output port. These
  48  * are the core display clock (CDCLK) and RAWCLK.
  49  *
  50  * CDCLK clocks most of the display pipe logic, and thus its frequency
  51  * must be high enough to support the rate at which pixels are flowing
  52  * through the pipes. Downscaling must also be accounted as that increases
  53  * the effective pixel rate.
  54  *
  55  * On several platforms the CDCLK frequency can be changed dynamically
  56  * to minimize power consumption for a given display configuration.
  57  * Typically changes to the CDCLK frequency require all the display pipes
  58  * to be shut down while the frequency is being changed.
  59  *
  60  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  61  * DMC will not change the active CDCLK frequency however, so that part
  62  * will still be performed by the driver directly.
  63  *
  64  * RAWCLK is a fixed frequency clock, often used by various auxiliary
  65  * blocks such as AUX CH or backlight PWM. Hence the only thing we
  66  * really need to know about RAWCLK is its frequency so that various
  67  * dividers can be programmed correctly.
  68  */
  69
  70 struct intel_cdclk_funcs {
  71         void (*get_cdclk)(struct drm_i915_private *i915,
  72                           struct intel_cdclk_config *cdclk_config);
  73         void (*set_cdclk)(struct drm_i915_private *i915,
  74                           const struct intel_cdclk_config *cdclk_config,
  75                           enum pipe pipe);
  76         int (*modeset_calc_cdclk)(struct intel_cdclk_state *state);
  77         u8 (*calc_voltage_level)(int cdclk);
  78 };
  79
  80 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
  81                            struct intel_cdclk_config *cdclk_config)
  82 {
  83         dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config);
  84 }
  85
  86 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
  87                                   const struct intel_cdclk_config *cdclk_config,
  88                                   enum pipe pipe)
  89 {
  90         dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe);
  91 }
  92
  93 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
  94                                           struct intel_cdclk_state *cdclk_config)
  95 {
  96         return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config);
  97 }
  98
  99 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
 100                                          int cdclk)
 101 {
 102         return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk);
 103 }
 104
 105 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
 106                                    struct intel_cdclk_config *cdclk_config)
 107 {
 108         cdclk_config->cdclk = 133333;
 109 }
 110
 111 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
 112                                    struct intel_cdclk_config *cdclk_config)
 113 {
 114         cdclk_config->cdclk = 200000;
 115 }
 116
 117 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
 118                                    struct intel_cdclk_config *cdclk_config)
 119 {
 120         cdclk_config->cdclk = 266667;
 121 }
 122
 123 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
 124                                    struct intel_cdclk_config *cdclk_config)
 125 {
 126         cdclk_config->cdclk = 333333;
 127 }
 128
 129 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
 130                                    struct intel_cdclk_config *cdclk_config)
 131 {
 132         cdclk_config->cdclk = 400000;
 133 }
 134
 135 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
 136                                    struct intel_cdclk_config *cdclk_config)
 137 {
 138         cdclk_config->cdclk = 450000;
 139 }
 140
 141 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
 142                            struct intel_cdclk_config *cdclk_config)
 143 {
 144         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 145         u16 hpllcc = 0;
 146
 147         /*
 148          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 149          * encoding is different :(
 150          * FIXME is this the right way to detect 852GM/852GMV?
 151          */
 152         if (pdev->revision == 0x1) {
 153                 cdclk_config->cdclk = 133333;
 154                 return;
 155         }
 156
 157         pci_bus_read_config_word(pdev->bus,
 158                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 159
 160         /* Assume that the hardware is in the high speed state.  This
 161          * should be the default.
 162          */
 163         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 164         case GC_CLOCK_133_200:
 165         case GC_CLOCK_133_200_2:
 166         case GC_CLOCK_100_200:
 167                 cdclk_config->cdclk = 200000;
 168                 break;
 169         case GC_CLOCK_166_250:
 170                 cdclk_config->cdclk = 250000;
 171                 break;
 172         case GC_CLOCK_100_133:
 173                 cdclk_config->cdclk = 133333;
 174                 break;
 175         case GC_CLOCK_133_266:
 176         case GC_CLOCK_133_266_2:
 177         case GC_CLOCK_166_266:
 178                 cdclk_config->cdclk = 266667;
 179                 break;
 180         }
 181 }
 182
 183 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 184                              struct intel_cdclk_config *cdclk_config)
 185 {
 186         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 187         u16 gcfgc = 0;
 188
 189         pci_read_config_word(pdev, GCFGC, &gcfgc);
 190
 191         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 192                 cdclk_config->cdclk = 133333;
 193                 return;
 194         }
 195
 196         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 197         case GC_DISPLAY_CLOCK_333_320_MHZ:
 198                 cdclk_config->cdclk = 333333;
 199                 break;
 200         default:
 201         case GC_DISPLAY_CLOCK_190_200_MHZ:
 202                 cdclk_config->cdclk = 190000;
 203                 break;
 204         }
 205 }
 206
 207 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 208                              struct intel_cdclk_config *cdclk_config)
 209 {
 210         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 211         u16 gcfgc = 0;
 212
 213         pci_read_config_word(pdev, GCFGC, &gcfgc);
 214
 215         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 216                 cdclk_config->cdclk = 133333;
 217                 return;
 218         }
 219
 220         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 221         case GC_DISPLAY_CLOCK_333_320_MHZ:
 222                 cdclk_config->cdclk = 320000;
 223                 break;
 224         default:
 225         case GC_DISPLAY_CLOCK_190_200_MHZ:
 226                 cdclk_config->cdclk = 200000;
 227                 break;
 228         }
 229 }
 230
 231 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 232 {
 233         static const unsigned int blb_vco[8] = {
 234                 [0] = 3200000,
 235                 [1] = 4000000,
 236                 [2] = 5333333,
 237                 [3] = 4800000,
 238                 [4] = 6400000,
 239         };
 240         static const unsigned int pnv_vco[8] = {
 241                 [0] = 3200000,
 242                 [1] = 4000000,
 243                 [2] = 5333333,
 244                 [3] = 4800000,
 245                 [4] = 2666667,
 246         };
 247         static const unsigned int cl_vco[8] = {
 248                 [0] = 3200000,
 249                 [1] = 4000000,
 250                 [2] = 5333333,
 251                 [3] = 6400000,
 252                 [4] = 3333333,
 253                 [5] = 3566667,
 254                 [6] = 4266667,
 255         };
 256         static const unsigned int elk_vco[8] = {
 257                 [0] = 3200000,
 258                 [1] = 4000000,
 259                 [2] = 5333333,
 260                 [3] = 4800000,
 261         };
 262         static const unsigned int ctg_vco[8] = {
 263                 [0] = 3200000,
 264                 [1] = 4000000,
 265                 [2] = 5333333,
 266                 [3] = 6400000,
 267                 [4] = 2666667,
 268                 [5] = 4266667,
 269         };
 270         const unsigned int *vco_table;
 271         unsigned int vco;
 272         u8 tmp = 0;
 273
 274         /* FIXME other chipsets? */
 275         if (IS_GM45(dev_priv))
 276                 vco_table = ctg_vco;
 277         else if (IS_G45(dev_priv))
 278                 vco_table = elk_vco;
 279         else if (IS_I965GM(dev_priv))
 280                 vco_table = cl_vco;
 281         else if (IS_PINEVIEW(dev_priv))
 282                 vco_table = pnv_vco;
 283         else if (IS_G33(dev_priv))
 284                 vco_table = blb_vco;
 285         else
 286                 return 0;
 287
 288         tmp = intel_de_read(dev_priv,
 289                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 290
 291         vco = vco_table[tmp & 0x7];
 292         if (vco == 0)
 293                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 294                         tmp);
 295         else
 296                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 297
 298         return vco;
 299 }
 300
 301 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 302                           struct intel_cdclk_config *cdclk_config)
 303 {
 304         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 305         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 306         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 307         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 308         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 309         const u8 *div_table;
 310         unsigned int cdclk_sel;
 311         u16 tmp = 0;
 312
 313         cdclk_config->vco = intel_hpll_vco(dev_priv);
 314
 315         pci_read_config_word(pdev, GCFGC, &tmp);
 316
 317         cdclk_sel = (tmp >> 4) & 0x7;
 318
 319         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 320                 goto fail;
 321
 322         switch (cdclk_config->vco) {
 323         case 3200000:
 324                 div_table = div_3200;
 325                 break;
 326         case 4000000:
 327                 div_table = div_4000;
 328                 break;
 329         case 4800000:
 330                 div_table = div_4800;
 331                 break;
 332         case 5333333:
 333                 div_table = div_5333;
 334                 break;
 335         default:
 336                 goto fail;
 337         }
 338
 339         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 340                                                 div_table[cdclk_sel]);
 341         return;
 342
 343 fail:
 344         drm_err(&dev_priv->drm,
 345                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 346                 cdclk_config->vco, tmp);
 347         cdclk_config->cdclk = 190476;
 348 }
 349
 350 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 351                           struct intel_cdclk_config *cdclk_config)
 352 {
 353         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 354         u16 gcfgc = 0;
 355
 356         pci_read_config_word(pdev, GCFGC, &gcfgc);
 357
 358         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 359         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 360                 cdclk_config->cdclk = 266667;
 361                 break;
 362         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 363                 cdclk_config->cdclk = 333333;
 364                 break;
 365         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 366                 cdclk_config->cdclk = 444444;
 367                 break;
 368         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 369                 cdclk_config->cdclk = 200000;
 370                 break;
 371         default:
 372                 drm_err(&dev_priv->drm,
 373                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 374                 fallthrough;
 375         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 376                 cdclk_config->cdclk = 133333;
 377                 break;
 378         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 379                 cdclk_config->cdclk = 166667;
 380                 break;
 381         }
 382 }
 383
 384 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 385                              struct intel_cdclk_config *cdclk_config)
 386 {
 387         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 388         static const u8 div_3200[] = { 16, 10,  8 };
 389         static const u8 div_4000[] = { 20, 12, 10 };
 390         static const u8 div_5333[] = { 24, 16, 14 };
 391         const u8 *div_table;
 392         unsigned int cdclk_sel;
 393         u16 tmp = 0;
 394
 395         cdclk_config->vco = intel_hpll_vco(dev_priv);
 396
 397         pci_read_config_word(pdev, GCFGC, &tmp);
 398
 399         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 400
 401         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 402                 goto fail;
 403
 404         switch (cdclk_config->vco) {
 405         case 3200000:
 406                 div_table = div_3200;
 407                 break;
 408         case 4000000:
 409                 div_table = div_4000;
 410                 break;
 411         case 5333333:
 412                 div_table = div_5333;
 413                 break;
 414         default:
 415                 goto fail;
 416         }
 417
 418         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 419                                                 div_table[cdclk_sel]);
 420         return;
 421
 422 fail:
 423         drm_err(&dev_priv->drm,
 424                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 425                 cdclk_config->vco, tmp);
 426         cdclk_config->cdclk = 200000;
 427 }
 428
 429 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 430                            struct intel_cdclk_config *cdclk_config)
 431 {
 432         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 433         unsigned int cdclk_sel;
 434         u16 tmp = 0;
 435
 436         cdclk_config->vco = intel_hpll_vco(dev_priv);
 437
 438         pci_read_config_word(pdev, GCFGC, &tmp);
 439
 440         cdclk_sel = (tmp >> 12) & 0x1;
 441
 442         switch (cdclk_config->vco) {
 443         case 2666667:
 444         case 4000000:
 445         case 5333333:
 446                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 447                 break;
 448         case 3200000:
 449                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 450                 break;
 451         default:
 452                 drm_err(&dev_priv->drm,
 453                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 454                         cdclk_config->vco, tmp);
 455                 cdclk_config->cdclk = 222222;
 456                 break;
 457         }
 458 }
 459
 460 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 461                           struct intel_cdclk_config *cdclk_config)
 462 {
 463         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 464         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 465
 466         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 467                 cdclk_config->cdclk = 800000;
 468         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 469                 cdclk_config->cdclk = 450000;
 470         else if (freq == LCPLL_CLK_FREQ_450)
 471                 cdclk_config->cdclk = 450000;
 472         else if (IS_HSW_ULT(dev_priv))
 473                 cdclk_config->cdclk = 337500;
 474         else
 475                 cdclk_config->cdclk = 540000;
 476 }
 477
 478 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 479 {
 480         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 481                 333333 : 320000;
 482
 483         /*
 484          * We seem to get an unstable or solid color picture at 200MHz.
 485          * Not sure what's wrong. For now use 200MHz only when all pipes
 486          * are off.
 487          */
 488         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 489                 return 400000;
 490         else if (min_cdclk > 266667)
 491                 return freq_320;
 492         else if (min_cdclk > 0)
 493                 return 266667;
 494         else
 495                 return 200000;
 496 }
 497
 498 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 499 {
 500         if (IS_VALLEYVIEW(dev_priv)) {
 501                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 502                         return 2;
 503                 else if (cdclk >= 266667)
 504                         return 1;
 505                 else
 506                         return 0;
 507         } else {
 508                 /*
 509                  * Specs are full of misinformation, but testing on actual
 510                  * hardware has shown that we just need to write the desired
 511                  * CCK divider into the Punit register.
 512                  */
 513                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 514         }
 515 }
 516
 517 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 518                           struct intel_cdclk_config *cdclk_config)
 519 {
 520         u32 val;
 521
 522         vlv_iosf_sb_get(dev_priv,
 523                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 524
 525         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 526         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 527                                                 CCK_DISPLAY_CLOCK_CONTROL,
 528                                                 cdclk_config->vco);
 529
 530         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 531
 532         vlv_iosf_sb_put(dev_priv,
 533                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 534
 535         if (IS_VALLEYVIEW(dev_priv))
 536                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 537                         DSPFREQGUAR_SHIFT;
 538         else
 539                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 540                         DSPFREQGUAR_SHIFT_CHV;
 541 }
 542
 543 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 544 {
 545         unsigned int credits, default_credits;
 546
 547         if (IS_CHERRYVIEW(dev_priv))
 548                 default_credits = PFI_CREDIT(12);
 549         else
 550                 default_credits = PFI_CREDIT(8);
 551
 552         if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 553                 /* CHV suggested value is 31 or 63 */
 554                 if (IS_CHERRYVIEW(dev_priv))
 555                         credits = PFI_CREDIT_63;
 556                 else
 557                         credits = PFI_CREDIT(15);
 558         } else {
 559                 credits = default_credits;
 560         }
 561
 562         /*
 563          * WA - write default credits before re-programming
 564          * FIXME: should we also set the resend bit here?
 565          */
 566         intel_de_write(dev_priv, GCI_CONTROL,
 567                        VGA_FAST_MODE_DISABLE | default_credits);
 568
 569         intel_de_write(dev_priv, GCI_CONTROL,
 570                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 571
 572         /*
 573          * FIXME is this guaranteed to clear
 574          * immediately or should we poll for it?
 575          */
 576         drm_WARN_ON(&dev_priv->drm,
 577                     intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 578 }
 579
 580 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 581                           const struct intel_cdclk_config *cdclk_config,
 582                           enum pipe pipe)
 583 {
 584         int cdclk = cdclk_config->cdclk;
 585         u32 val, cmd = cdclk_config->voltage_level;
 586         intel_wakeref_t wakeref;
 587
 588         switch (cdclk) {
 589         case 400000:
 590         case 333333:
 591         case 320000:
 592         case 266667:
 593         case 200000:
 594                 break;
 595         default:
 596                 MISSING_CASE(cdclk);
 597                 return;
 598         }
 599
 600         /* There are cases where we can end up here with power domains
 601          * off and a CDCLK frequency other than the minimum, like when
 602          * issuing a modeset without actually changing any display after
 603          * a system suspend.  So grab the display core domain, which covers
 604          * the HW blocks needed for the following programming.
 605          */
 606         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 607
 608         vlv_iosf_sb_get(dev_priv,
 609                         BIT(VLV_IOSF_SB_CCK) |
 610                         BIT(VLV_IOSF_SB_BUNIT) |
 611                         BIT(VLV_IOSF_SB_PUNIT));
 612
 613         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 614         val &= ~DSPFREQGUAR_MASK;
 615         val |= (cmd << DSPFREQGUAR_SHIFT);
 616         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 617         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 618                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 619                      50)) {
 620                 drm_err(&dev_priv->drm,
 621                         "timed out waiting for CDclk change\n");
 622         }
 623
 624         if (cdclk == 400000) {
 625                 u32 divider;
 626
 627                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 628                                             cdclk) - 1;
 629
 630                 /* adjust cdclk divider */
 631                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 632                 val &= ~CCK_FREQUENCY_VALUES;
 633                 val |= divider;
 634                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 635
 636                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 637                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 638                              50))
 639                         drm_err(&dev_priv->drm,
 640                                 "timed out waiting for CDclk change\n");
 641         }
 642
 643         /* adjust self-refresh exit latency value */
 644         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 645         val &= ~0x7f;
 646
 647         /*
 648          * For high bandwidth configs, we set a higher latency in the bunit
 649          * so that the core display fetch happens in time to avoid underruns.
 650          */
 651         if (cdclk == 400000)
 652                 val |= 4500 / 250; /* 4.5 usec */
 653         else
 654                 val |= 3000 / 250; /* 3.0 usec */
 655         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 656
 657         vlv_iosf_sb_put(dev_priv,
 658                         BIT(VLV_IOSF_SB_CCK) |
 659                         BIT(VLV_IOSF_SB_BUNIT) |
 660                         BIT(VLV_IOSF_SB_PUNIT));
 661
 662         intel_update_cdclk(dev_priv);
 663
 664         vlv_program_pfi_credits(dev_priv);
 665
 666         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 667 }
 668
 669 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 670                           const struct intel_cdclk_config *cdclk_config,
 671                           enum pipe pipe)
 672 {
 673         int cdclk = cdclk_config->cdclk;
 674         u32 val, cmd = cdclk_config->voltage_level;
 675         intel_wakeref_t wakeref;
 676
 677         switch (cdclk) {
 678         case 333333:
 679         case 320000:
 680         case 266667:
 681         case 200000:
 682                 break;
 683         default:
 684                 MISSING_CASE(cdclk);
 685                 return;
 686         }
 687
 688         /* There are cases where we can end up here with power domains
 689          * off and a CDCLK frequency other than the minimum, like when
 690          * issuing a modeset without actually changing any display after
 691          * a system suspend.  So grab the display core domain, which covers
 692          * the HW blocks needed for the following programming.
 693          */
 694         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 695
 696         vlv_punit_get(dev_priv);
 697         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 698         val &= ~DSPFREQGUAR_MASK_CHV;
 699         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 700         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 701         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 702                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 703                      50)) {
 704                 drm_err(&dev_priv->drm,
 705                         "timed out waiting for CDclk change\n");
 706         }
 707
 708         vlv_punit_put(dev_priv);
 709
 710         intel_update_cdclk(dev_priv);
 711
 712         vlv_program_pfi_credits(dev_priv);
 713
 714         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 715 }
 716
 717 static int bdw_calc_cdclk(int min_cdclk)
 718 {
 719         if (min_cdclk > 540000)
 720                 return 675000;
 721         else if (min_cdclk > 450000)
 722                 return 540000;
 723         else if (min_cdclk > 337500)
 724                 return 450000;
 725         else
 726                 return 337500;
 727 }
 728
 729 static u8 bdw_calc_voltage_level(int cdclk)
 730 {
 731         switch (cdclk) {
 732         default:
 733         case 337500:
 734                 return 2;
 735         case 450000:
 736                 return 0;
 737         case 540000:
 738                 return 1;
 739         case 675000:
 740                 return 3;
 741         }
 742 }
 743
 744 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 745                           struct intel_cdclk_config *cdclk_config)
 746 {
 747         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 748         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 749
 750         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 751                 cdclk_config->cdclk = 800000;
 752         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 753                 cdclk_config->cdclk = 450000;
 754         else if (freq == LCPLL_CLK_FREQ_450)
 755                 cdclk_config->cdclk = 450000;
 756         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 757                 cdclk_config->cdclk = 540000;
 758         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 759                 cdclk_config->cdclk = 337500;
 760         else
 761                 cdclk_config->cdclk = 675000;
 762
 763         /*
 764          * Can't read this out :( Let's assume it's
 765          * at least what the CDCLK frequency requires.
 766          */
 767         cdclk_config->voltage_level =
 768                 bdw_calc_voltage_level(cdclk_config->cdclk);
 769 }
 770
 771 static u32 bdw_cdclk_freq_sel(int cdclk)
 772 {
 773         switch (cdclk) {
 774         default:
 775                 MISSING_CASE(cdclk);
 776                 fallthrough;
 777         case 337500:
 778                 return LCPLL_CLK_FREQ_337_5_BDW;
 779         case 450000:
 780                 return LCPLL_CLK_FREQ_450;
 781         case 540000:
 782                 return LCPLL_CLK_FREQ_54O_BDW;
 783         case 675000:
 784                 return LCPLL_CLK_FREQ_675_BDW;
 785         }
 786 }
 787
 788 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 789                           const struct intel_cdclk_config *cdclk_config,
 790                           enum pipe pipe)
 791 {
 792         int cdclk = cdclk_config->cdclk;
 793         int ret;
 794
 795         if (drm_WARN(&dev_priv->drm,
 796                      (intel_de_read(dev_priv, LCPLL_CTL) &
 797                       (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 798                        LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 799                        LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 800                        LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 801                      "trying to change cdclk frequency with cdclk not enabled\n"))
 802                 return;
 803
 804         ret = snb_pcode_write(&dev_priv->uncore, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 805         if (ret) {
 806                 drm_err(&dev_priv->drm,
 807                         "failed to inform pcode about cdclk change\n");
 808                 return;
 809         }
 810
 811         intel_de_rmw(dev_priv, LCPLL_CTL,
 812                      0, LCPLL_CD_SOURCE_FCLK);
 813
 814         /*
 815          * According to the spec, it should be enough to poll for this 1 us.
 816          * However, extensive testing shows that this can take longer.
 817          */
 818         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 819                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 820                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 821
 822         intel_de_rmw(dev_priv, LCPLL_CTL,
 823                      LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 824
 825         intel_de_rmw(dev_priv, LCPLL_CTL,
 826                      LCPLL_CD_SOURCE_FCLK, 0);
 827
 828         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 829                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 830                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 831
 832         snb_pcode_write(&dev_priv->uncore, HSW_PCODE_DE_WRITE_FREQ_REQ,
 833                         cdclk_config->voltage_level);
 834
 835         intel_de_write(dev_priv, CDCLK_FREQ,
 836                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 837
 838         intel_update_cdclk(dev_priv);
 839 }
 840
 841 static int skl_calc_cdclk(int min_cdclk, int vco)
 842 {
 843         if (vco == 8640000) {
 844                 if (min_cdclk > 540000)
 845                         return 617143;
 846                 else if (min_cdclk > 432000)
 847                         return 540000;
 848                 else if (min_cdclk > 308571)
 849                         return 432000;
 850                 else
 851                         return 308571;
 852         } else {
 853                 if (min_cdclk > 540000)
 854                         return 675000;
 855                 else if (min_cdclk > 450000)
 856                         return 540000;
 857                 else if (min_cdclk > 337500)
 858                         return 450000;
 859                 else
 860                         return 337500;
 861         }
 862 }
 863
 864 static u8 skl_calc_voltage_level(int cdclk)
 865 {
 866         if (cdclk > 540000)
 867                 return 3;
 868         else if (cdclk > 450000)
 869                 return 2;
 870         else if (cdclk > 337500)
 871                 return 1;
 872         else
 873                 return 0;
 874 }
 875
 876 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 877                              struct intel_cdclk_config *cdclk_config)
 878 {
 879         u32 val;
 880
 881         cdclk_config->ref = 24000;
 882         cdclk_config->vco = 0;
 883
 884         val = intel_de_read(dev_priv, LCPLL1_CTL);
 885         if ((val & LCPLL_PLL_ENABLE) == 0)
 886                 return;
 887
 888         if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 889                 return;
 890
 891         val = intel_de_read(dev_priv, DPLL_CTRL1);
 892
 893         if (drm_WARN_ON(&dev_priv->drm,
 894                         (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 895                                 DPLL_CTRL1_SSC(SKL_DPLL0) |
 896                                 DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 897                         DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 898                 return;
 899
 900         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 901         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 902         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 903         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 904         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 905                 cdclk_config->vco = 8100000;
 906                 break;
 907         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 908         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 909                 cdclk_config->vco = 8640000;
 910                 break;
 911         default:
 912                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 913                 break;
 914         }
 915 }
 916
 917 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 918                           struct intel_cdclk_config *cdclk_config)
 919 {
 920         u32 cdctl;
 921
 922         skl_dpll0_update(dev_priv, cdclk_config);
 923
 924         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 925
 926         if (cdclk_config->vco == 0)
 927                 goto out;
 928
 929         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 930
 931         if (cdclk_config->vco == 8640000) {
 932                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 933                 case CDCLK_FREQ_450_432:
 934                         cdclk_config->cdclk = 432000;
 935                         break;
 936                 case CDCLK_FREQ_337_308:
 937                         cdclk_config->cdclk = 308571;
 938                         break;
 939                 case CDCLK_FREQ_540:
 940                         cdclk_config->cdclk = 540000;
 941                         break;
 942                 case CDCLK_FREQ_675_617:
 943                         cdclk_config->cdclk = 617143;
 944                         break;
 945                 default:
 946                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 947                         break;
 948                 }
 949         } else {
 950                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 951                 case CDCLK_FREQ_450_432:
 952                         cdclk_config->cdclk = 450000;
 953                         break;
 954                 case CDCLK_FREQ_337_308:
 955                         cdclk_config->cdclk = 337500;
 956                         break;
 957                 case CDCLK_FREQ_540:
 958                         cdclk_config->cdclk = 540000;
 959                         break;
 960                 case CDCLK_FREQ_675_617:
 961                         cdclk_config->cdclk = 675000;
 962                         break;
 963                 default:
 964                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 965                         break;
 966                 }
 967         }
 968
 969  out:
 970         /*
 971          * Can't read this out :( Let's assume it's
 972          * at least what the CDCLK frequency requires.
 973          */
 974         cdclk_config->voltage_level =
 975                 skl_calc_voltage_level(cdclk_config->cdclk);
 976 }
 977
 978 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
 979 static int skl_cdclk_decimal(int cdclk)
 980 {
 981         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 982 }
 983
 984 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
 985                                         int vco)
 986 {
 987         bool changed = dev_priv->skl_preferred_vco_freq != vco;
 988
 989         dev_priv->skl_preferred_vco_freq = vco;
 990
 991         if (changed)
 992                 intel_update_max_cdclk(dev_priv);
 993 }
 994
 995 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
 996 {
 997         drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
 998
 999         /*
1000          * We always enable DPLL0 with the lowest link rate possible, but still
1001          * taking into account the VCO required to operate the eDP panel at the
1002          * desired frequency. The usual DP link rates operate with a VCO of
1003          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
1004          * The modeset code is responsible for the selection of the exact link
1005          * rate later on, with the constraint of choosing a frequency that
1006          * works with vco.
1007          */
1008         if (vco == 8640000)
1009                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1010         else
1011                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1012 }
1013
1014 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1015 {
1016         intel_de_rmw(dev_priv, DPLL_CTRL1,
1017                      DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1018                      DPLL_CTRL1_SSC(SKL_DPLL0) |
1019                      DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1020                      DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1021                      skl_dpll0_link_rate(dev_priv, vco));
1022         intel_de_posting_read(dev_priv, DPLL_CTRL1);
1023
1024         intel_de_rmw(dev_priv, LCPLL1_CTL,
1025                      0, LCPLL_PLL_ENABLE);
1026
1027         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1028                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1029
1030         dev_priv->display.cdclk.hw.vco = vco;
1031
1032         /* We'll want to keep using the current vco from now on. */
1033         skl_set_preferred_cdclk_vco(dev_priv, vco);
1034 }
1035
1036 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1037 {
1038         intel_de_rmw(dev_priv, LCPLL1_CTL,
1039                      LCPLL_PLL_ENABLE, 0);
1040
1041         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1042                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1043
1044         dev_priv->display.cdclk.hw.vco = 0;
1045 }
1046
1047 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1048                               int cdclk, int vco)
1049 {
1050         switch (cdclk) {
1051         default:
1052                 drm_WARN_ON(&dev_priv->drm,
1053                             cdclk != dev_priv->display.cdclk.hw.bypass);
1054                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1055                 fallthrough;
1056         case 308571:
1057         case 337500:
1058                 return CDCLK_FREQ_337_308;
1059         case 450000:
1060         case 432000:
1061                 return CDCLK_FREQ_450_432;
1062         case 540000:
1063                 return CDCLK_FREQ_540;
1064         case 617143:
1065         case 675000:
1066                 return CDCLK_FREQ_675_617;
1067         }
1068 }
1069
1070 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1071                           const struct intel_cdclk_config *cdclk_config,
1072                           enum pipe pipe)
1073 {
1074         int cdclk = cdclk_config->cdclk;
1075         int vco = cdclk_config->vco;
1076         u32 freq_select, cdclk_ctl;
1077         int ret;
1078
1079         /*
1080          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1081          * unsupported on SKL. In theory this should never happen since only
1082          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1083          * supported on SKL either, see the above WA. WARN whenever trying to
1084          * use the corresponding VCO freq as that always leads to using the
1085          * minimum 308MHz CDCLK.
1086          */
1087         drm_WARN_ON_ONCE(&dev_priv->drm,
1088                          IS_SKYLAKE(dev_priv) && vco == 8640000);
1089
1090         ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1091                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1092                                 SKL_CDCLK_READY_FOR_CHANGE,
1093                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1094         if (ret) {
1095                 drm_err(&dev_priv->drm,
1096                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1097                 return;
1098         }
1099
1100         freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1101
1102         if (dev_priv->display.cdclk.hw.vco != 0 &&
1103             dev_priv->display.cdclk.hw.vco != vco)
1104                 skl_dpll0_disable(dev_priv);
1105
1106         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1107
1108         if (dev_priv->display.cdclk.hw.vco != vco) {
1109                 /* Wa Display #1183: skl,kbl,cfl */
1110                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1111                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1112                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1113         }
1114
1115         /* Wa Display #1183: skl,kbl,cfl */
1116         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1117         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1118         intel_de_posting_read(dev_priv, CDCLK_CTL);
1119
1120         if (dev_priv->display.cdclk.hw.vco != vco)
1121                 skl_dpll0_enable(dev_priv, vco);
1122
1123         /* Wa Display #1183: skl,kbl,cfl */
1124         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1125         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1126
1127         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1128         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1129
1130         /* Wa Display #1183: skl,kbl,cfl */
1131         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1132         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1133         intel_de_posting_read(dev_priv, CDCLK_CTL);
1134
1135         /* inform PCU of the change */
1136         snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1137                         cdclk_config->voltage_level);
1138
1139         intel_update_cdclk(dev_priv);
1140 }
1141
1142 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1143 {
1144         u32 cdctl, expected;
1145
1146         /*
1147          * check if the pre-os initialized the display
1148          * There is SWF18 scratchpad register defined which is set by the
1149          * pre-os which can be used by the OS drivers to check the status
1150          */
1151         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1152                 goto sanitize;
1153
1154         intel_update_cdclk(dev_priv);
1155         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1156
1157         /* Is PLL enabled and locked ? */
1158         if (dev_priv->display.cdclk.hw.vco == 0 ||
1159             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1160                 goto sanitize;
1161
1162         /* DPLL okay; verify the cdclock
1163          *
1164          * Noticed in some instances that the freq selection is correct but
1165          * decimal part is programmed wrong from BIOS where pre-os does not
1166          * enable display. Verify the same as well.
1167          */
1168         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1169         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1170                 skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk);
1171         if (cdctl == expected)
1172                 /* All well; nothing to sanitize */
1173                 return;
1174
1175 sanitize:
1176         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1177
1178         /* force cdclk programming */
1179         dev_priv->display.cdclk.hw.cdclk = 0;
1180         /* force full PLL disable + enable */
1181         dev_priv->display.cdclk.hw.vco = -1;
1182 }
1183
1184 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1185 {
1186         struct intel_cdclk_config cdclk_config;
1187
1188         skl_sanitize_cdclk(dev_priv);
1189
1190         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1191             dev_priv->display.cdclk.hw.vco != 0) {
1192                 /*
1193                  * Use the current vco as our initial
1194                  * guess as to what the preferred vco is.
1195                  */
1196                 if (dev_priv->skl_preferred_vco_freq == 0)
1197                         skl_set_preferred_cdclk_vco(dev_priv,
1198                                                     dev_priv->display.cdclk.hw.vco);
1199                 return;
1200         }
1201
1202         cdclk_config = dev_priv->display.cdclk.hw;
1203
1204         cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1205         if (cdclk_config.vco == 0)
1206                 cdclk_config.vco = 8100000;
1207         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1208         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1209
1210         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1211 }
1212
1213 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1214 {
1215         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1216
1217         cdclk_config.cdclk = cdclk_config.bypass;
1218         cdclk_config.vco = 0;
1219         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1220
1221         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1222 }
1223
1224 struct intel_cdclk_vals {
1225         u32 cdclk;
1226         u16 refclk;
1227         u16 waveform;
1228         u8 divider;     /* CD2X divider * 2 */
1229         u8 ratio;
1230 };
1231
1232 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1233         { .refclk = 19200, .cdclk = 144000, .divider = 8, .ratio = 60 },
1234         { .refclk = 19200, .cdclk = 288000, .divider = 4, .ratio = 60 },
1235         { .refclk = 19200, .cdclk = 384000, .divider = 3, .ratio = 60 },
1236         { .refclk = 19200, .cdclk = 576000, .divider = 2, .ratio = 60 },
1237         { .refclk = 19200, .cdclk = 624000, .divider = 2, .ratio = 65 },
1238         {}
1239 };
1240
1241 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1242         { .refclk = 19200, .cdclk =  79200, .divider = 8, .ratio = 33 },
1243         { .refclk = 19200, .cdclk = 158400, .divider = 4, .ratio = 33 },
1244         { .refclk = 19200, .cdclk = 316800, .divider = 2, .ratio = 33 },
1245         {}
1246 };
1247
1248 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1249         { .refclk = 19200, .cdclk = 172800, .divider = 2, .ratio = 18 },
1250         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1251         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1252         { .refclk = 19200, .cdclk = 326400, .divider = 4, .ratio = 68 },
1253         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1254         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1255
1256         { .refclk = 24000, .cdclk = 180000, .divider = 2, .ratio = 15 },
1257         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1258         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1259         { .refclk = 24000, .cdclk = 324000, .divider = 4, .ratio = 54 },
1260         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1261         { .refclk = 24000, .cdclk = 648000, .divider = 2, .ratio = 54 },
1262
1263         { .refclk = 38400, .cdclk = 172800, .divider = 2, .ratio =  9 },
1264         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1265         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1266         { .refclk = 38400, .cdclk = 326400, .divider = 4, .ratio = 34 },
1267         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1268         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1269         {}
1270 };
1271
1272 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1273         { .refclk = 19200, .cdclk = 172800, .divider = 4, .ratio =  36 },
1274         { .refclk = 19200, .cdclk = 192000, .divider = 4, .ratio =  40 },
1275         { .refclk = 19200, .cdclk = 307200, .divider = 4, .ratio =  64 },
1276         { .refclk = 19200, .cdclk = 326400, .divider = 8, .ratio = 136 },
1277         { .refclk = 19200, .cdclk = 556800, .divider = 4, .ratio = 116 },
1278         { .refclk = 19200, .cdclk = 652800, .divider = 4, .ratio = 136 },
1279
1280         { .refclk = 24000, .cdclk = 180000, .divider = 4, .ratio =  30 },
1281         { .refclk = 24000, .cdclk = 192000, .divider = 4, .ratio =  32 },
1282         { .refclk = 24000, .cdclk = 312000, .divider = 4, .ratio =  52 },
1283         { .refclk = 24000, .cdclk = 324000, .divider = 8, .ratio = 108 },
1284         { .refclk = 24000, .cdclk = 552000, .divider = 4, .ratio =  92 },
1285         { .refclk = 24000, .cdclk = 648000, .divider = 4, .ratio = 108 },
1286
1287         { .refclk = 38400, .cdclk = 172800, .divider = 4, .ratio = 18 },
1288         { .refclk = 38400, .cdclk = 192000, .divider = 4, .ratio = 20 },
1289         { .refclk = 38400, .cdclk = 307200, .divider = 4, .ratio = 32 },
1290         { .refclk = 38400, .cdclk = 326400, .divider = 8, .ratio = 68 },
1291         { .refclk = 38400, .cdclk = 556800, .divider = 4, .ratio = 58 },
1292         { .refclk = 38400, .cdclk = 652800, .divider = 4, .ratio = 68 },
1293         {}
1294 };
1295
1296 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1297         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1298         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1299         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1300
1301         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1302         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1303         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1304
1305         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1306         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1307         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1308         {}
1309 };
1310
1311 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1312         { .refclk = 19200, .cdclk = 172800, .divider = 3, .ratio = 27 },
1313         { .refclk = 19200, .cdclk = 192000, .divider = 2, .ratio = 20 },
1314         { .refclk = 19200, .cdclk = 307200, .divider = 2, .ratio = 32 },
1315         { .refclk = 19200, .cdclk = 556800, .divider = 2, .ratio = 58 },
1316         { .refclk = 19200, .cdclk = 652800, .divider = 2, .ratio = 68 },
1317
1318         { .refclk = 24000, .cdclk = 176000, .divider = 3, .ratio = 22 },
1319         { .refclk = 24000, .cdclk = 192000, .divider = 2, .ratio = 16 },
1320         { .refclk = 24000, .cdclk = 312000, .divider = 2, .ratio = 26 },
1321         { .refclk = 24000, .cdclk = 552000, .divider = 2, .ratio = 46 },
1322         { .refclk = 24400, .cdclk = 648000, .divider = 2, .ratio = 54 },
1323
1324         { .refclk = 38400, .cdclk = 179200, .divider = 3, .ratio = 14 },
1325         { .refclk = 38400, .cdclk = 192000, .divider = 2, .ratio = 10 },
1326         { .refclk = 38400, .cdclk = 307200, .divider = 2, .ratio = 16 },
1327         { .refclk = 38400, .cdclk = 556800, .divider = 2, .ratio = 29 },
1328         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34 },
1329         {}
1330 };
1331
1332 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1333         { .refclk = 38400, .cdclk = 163200, .divider = 2, .ratio = 34, .waveform = 0x8888 },
1334         { .refclk = 38400, .cdclk = 204000, .divider = 2, .ratio = 34, .waveform = 0x9248 },
1335         { .refclk = 38400, .cdclk = 244800, .divider = 2, .ratio = 34, .waveform = 0xa4a4 },
1336         { .refclk = 38400, .cdclk = 285600, .divider = 2, .ratio = 34, .waveform = 0xa54a },
1337         { .refclk = 38400, .cdclk = 326400, .divider = 2, .ratio = 34, .waveform = 0xaaaa },
1338         { .refclk = 38400, .cdclk = 367200, .divider = 2, .ratio = 34, .waveform = 0xad5a },
1339         { .refclk = 38400, .cdclk = 408000, .divider = 2, .ratio = 34, .waveform = 0xb6b6 },
1340         { .refclk = 38400, .cdclk = 448800, .divider = 2, .ratio = 34, .waveform = 0xdbb6 },
1341         { .refclk = 38400, .cdclk = 489600, .divider = 2, .ratio = 34, .waveform = 0xeeee },
1342         { .refclk = 38400, .cdclk = 530400, .divider = 2, .ratio = 34, .waveform = 0xf7de },
1343         { .refclk = 38400, .cdclk = 571200, .divider = 2, .ratio = 34, .waveform = 0xfefe },
1344         { .refclk = 38400, .cdclk = 612000, .divider = 2, .ratio = 34, .waveform = 0xfffe },
1345         { .refclk = 38400, .cdclk = 652800, .divider = 2, .ratio = 34, .waveform = 0xffff },
1346         {}
1347 };
1348
1349 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1350 {
1351         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1352         int i;
1353
1354         for (i = 0; table[i].refclk; i++)
1355                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1356                     table[i].cdclk >= min_cdclk)
1357                         return table[i].cdclk;
1358
1359         drm_WARN(&dev_priv->drm, 1,
1360                  "Cannot satisfy minimum cdclk %d with refclk %u\n",
1361                  min_cdclk, dev_priv->display.cdclk.hw.ref);
1362         return 0;
1363 }
1364
1365 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1366 {
1367         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1368         int i;
1369
1370         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1371                 return 0;
1372
1373         for (i = 0; table[i].refclk; i++)
1374                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1375                     table[i].cdclk == cdclk)
1376                         return dev_priv->display.cdclk.hw.ref * table[i].ratio;
1377
1378         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1379                  cdclk, dev_priv->display.cdclk.hw.ref);
1380         return 0;
1381 }
1382
1383 static u8 bxt_calc_voltage_level(int cdclk)
1384 {
1385         return DIV_ROUND_UP(cdclk, 25000);
1386 }
1387
1388 static u8 icl_calc_voltage_level(int cdclk)
1389 {
1390         if (cdclk > 556800)
1391                 return 2;
1392         else if (cdclk > 312000)
1393                 return 1;
1394         else
1395                 return 0;
1396 }
1397
1398 static u8 ehl_calc_voltage_level(int cdclk)
1399 {
1400         if (cdclk > 326400)
1401                 return 3;
1402         else if (cdclk > 312000)
1403                 return 2;
1404         else if (cdclk > 180000)
1405                 return 1;
1406         else
1407                 return 0;
1408 }
1409
1410 static u8 tgl_calc_voltage_level(int cdclk)
1411 {
1412         if (cdclk > 556800)
1413                 return 3;
1414         else if (cdclk > 326400)
1415                 return 2;
1416         else if (cdclk > 312000)
1417                 return 1;
1418         else
1419                 return 0;
1420 }
1421
1422 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1423                                struct intel_cdclk_config *cdclk_config)
1424 {
1425         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1426
1427         switch (dssm) {
1428         default:
1429                 MISSING_CASE(dssm);
1430                 fallthrough;
1431         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1432                 cdclk_config->ref = 24000;
1433                 break;
1434         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1435                 cdclk_config->ref = 19200;
1436                 break;
1437         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1438                 cdclk_config->ref = 38400;
1439                 break;
1440         }
1441 }
1442
1443 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1444                                struct intel_cdclk_config *cdclk_config)
1445 {
1446         u32 val, ratio;
1447
1448         if (IS_DG2(dev_priv))
1449                 cdclk_config->ref = 38400;
1450         else if (DISPLAY_VER(dev_priv) >= 11)
1451                 icl_readout_refclk(dev_priv, cdclk_config);
1452         else
1453                 cdclk_config->ref = 19200;
1454
1455         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1456         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1457             (val & BXT_DE_PLL_LOCK) == 0) {
1458                 /*
1459                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1460                  * setting it to zero is a way to signal that.
1461                  */
1462                 cdclk_config->vco = 0;
1463                 return;
1464         }
1465
1466         /*
1467          * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1468          * gen9lp had it in a separate PLL control register.
1469          */
1470         if (DISPLAY_VER(dev_priv) >= 11)
1471                 ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1472         else
1473                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1474
1475         cdclk_config->vco = ratio * cdclk_config->ref;
1476 }
1477
1478 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1479                           struct intel_cdclk_config *cdclk_config)
1480 {
1481         u32 squash_ctl = 0;
1482         u32 divider;
1483         int div;
1484
1485         bxt_de_pll_readout(dev_priv, cdclk_config);
1486
1487         if (DISPLAY_VER(dev_priv) >= 12)
1488                 cdclk_config->bypass = cdclk_config->ref / 2;
1489         else if (DISPLAY_VER(dev_priv) >= 11)
1490                 cdclk_config->bypass = 50000;
1491         else
1492                 cdclk_config->bypass = cdclk_config->ref;
1493
1494         if (cdclk_config->vco == 0) {
1495                 cdclk_config->cdclk = cdclk_config->bypass;
1496                 goto out;
1497         }
1498
1499         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1500
1501         switch (divider) {
1502         case BXT_CDCLK_CD2X_DIV_SEL_1:
1503                 div = 2;
1504                 break;
1505         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1506                 div = 3;
1507                 break;
1508         case BXT_CDCLK_CD2X_DIV_SEL_2:
1509                 div = 4;
1510                 break;
1511         case BXT_CDCLK_CD2X_DIV_SEL_4:
1512                 div = 8;
1513                 break;
1514         default:
1515                 MISSING_CASE(divider);
1516                 return;
1517         }
1518
1519         if (HAS_CDCLK_SQUASH(dev_priv))
1520                 squash_ctl = intel_de_read(dev_priv, CDCLK_SQUASH_CTL);
1521
1522         if (squash_ctl & CDCLK_SQUASH_ENABLE) {
1523                 u16 waveform;
1524                 int size;
1525
1526                 size = REG_FIELD_GET(CDCLK_SQUASH_WINDOW_SIZE_MASK, squash_ctl) + 1;
1527                 waveform = REG_FIELD_GET(CDCLK_SQUASH_WAVEFORM_MASK, squash_ctl) >> (16 - size);
1528
1529                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(hweight16(waveform) *
1530                                                         cdclk_config->vco, size * div);
1531         } else {
1532                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1533         }
1534
1535  out:
1536         /*
1537          * Can't read this out :( Let's assume it's
1538          * at least what the CDCLK frequency requires.
1539          */
1540         cdclk_config->voltage_level =
1541                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1542 }
1543
1544 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1545 {
1546         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1547
1548         /* Timeout 200us */
1549         if (intel_de_wait_for_clear(dev_priv,
1550                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1551                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1552
1553         dev_priv->display.cdclk.hw.vco = 0;
1554 }
1555
1556 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1557 {
1558         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1559
1560         intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1561                      BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1562
1563         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1564
1565         /* Timeout 200us */
1566         if (intel_de_wait_for_set(dev_priv,
1567                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1568                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1569
1570         dev_priv->display.cdclk.hw.vco = vco;
1571 }
1572
1573 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1574 {
1575         intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1576                      BXT_DE_PLL_PLL_ENABLE, 0);
1577
1578         /* Timeout 200us */
1579         if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1580                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1581
1582         dev_priv->display.cdclk.hw.vco = 0;
1583 }
1584
1585 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1586 {
1587         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1588         u32 val;
1589
1590         val = ICL_CDCLK_PLL_RATIO(ratio);
1591         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1592
1593         val |= BXT_DE_PLL_PLL_ENABLE;
1594         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1595
1596         /* Timeout 200us */
1597         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1598                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1599
1600         dev_priv->display.cdclk.hw.vco = vco;
1601 }
1602
1603 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1604 {
1605         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1606         u32 val;
1607
1608         /* Write PLL ratio without disabling */
1609         val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1610         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1611
1612         /* Submit freq change request */
1613         val |= BXT_DE_PLL_FREQ_REQ;
1614         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1615
1616         /* Timeout 200us */
1617         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1618                                   BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1619                 drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n");
1620
1621         val &= ~BXT_DE_PLL_FREQ_REQ;
1622         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1623
1624         dev_priv->display.cdclk.hw.vco = vco;
1625 }
1626
1627 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1628 {
1629         if (DISPLAY_VER(dev_priv) >= 12) {
1630                 if (pipe == INVALID_PIPE)
1631                         return TGL_CDCLK_CD2X_PIPE_NONE;
1632                 else
1633                         return TGL_CDCLK_CD2X_PIPE(pipe);
1634         } else if (DISPLAY_VER(dev_priv) >= 11) {
1635                 if (pipe == INVALID_PIPE)
1636                         return ICL_CDCLK_CD2X_PIPE_NONE;
1637                 else
1638                         return ICL_CDCLK_CD2X_PIPE(pipe);
1639         } else {
1640                 if (pipe == INVALID_PIPE)
1641                         return BXT_CDCLK_CD2X_PIPE_NONE;
1642                 else
1643                         return BXT_CDCLK_CD2X_PIPE(pipe);
1644         }
1645 }
1646
1647 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1648                                   int cdclk, int vco)
1649 {
1650         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1651         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
1652         default:
1653                 drm_WARN_ON(&dev_priv->drm,
1654                             cdclk != dev_priv->display.cdclk.hw.bypass);
1655                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1656                 fallthrough;
1657         case 2:
1658                 return BXT_CDCLK_CD2X_DIV_SEL_1;
1659         case 3:
1660                 return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1661         case 4:
1662                 return BXT_CDCLK_CD2X_DIV_SEL_2;
1663         case 8:
1664                 return BXT_CDCLK_CD2X_DIV_SEL_4;
1665         }
1666 }
1667
1668 static u32 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
1669                                  int cdclk)
1670 {
1671         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1672         int i;
1673
1674         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1675                 return 0;
1676
1677         for (i = 0; table[i].refclk; i++)
1678                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1679                     table[i].cdclk == cdclk)
1680                         return table[i].waveform;
1681
1682         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1683                  cdclk, dev_priv->display.cdclk.hw.ref);
1684
1685         return 0xffff;
1686 }
1687
1688 static void icl_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1689 {
1690         if (i915->display.cdclk.hw.vco != 0 &&
1691             i915->display.cdclk.hw.vco != vco)
1692                 icl_cdclk_pll_disable(i915);
1693
1694         if (i915->display.cdclk.hw.vco != vco)
1695                 icl_cdclk_pll_enable(i915, vco);
1696 }
1697
1698 static void bxt_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1699 {
1700         if (i915->display.cdclk.hw.vco != 0 &&
1701             i915->display.cdclk.hw.vco != vco)
1702                 bxt_de_pll_disable(i915);
1703
1704         if (i915->display.cdclk.hw.vco != vco)
1705                 bxt_de_pll_enable(i915, vco);
1706 }
1707
1708 static void dg2_cdclk_squash_program(struct drm_i915_private *i915,
1709                                      u16 waveform)
1710 {
1711         u32 squash_ctl = 0;
1712
1713         if (waveform)
1714                 squash_ctl = CDCLK_SQUASH_ENABLE |
1715                              CDCLK_SQUASH_WINDOW_SIZE(0xf) | waveform;
1716
1717         intel_de_write(i915, CDCLK_SQUASH_CTL, squash_ctl);
1718 }
1719
1720 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
1721                           const struct intel_cdclk_config *cdclk_config,
1722                           enum pipe pipe)
1723 {
1724         int cdclk = cdclk_config->cdclk;
1725         int vco = cdclk_config->vco;
1726         u32 val;
1727         u16 waveform;
1728         int clock;
1729         int ret;
1730
1731         /* Inform power controller of upcoming frequency change. */
1732         if (DISPLAY_VER(dev_priv) >= 11)
1733                 ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1734                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
1735                                         SKL_CDCLK_READY_FOR_CHANGE,
1736                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
1737         else
1738                 /*
1739                  * BSpec requires us to wait up to 150usec, but that leads to
1740                  * timeouts; the 2ms used here is based on experiment.
1741                  */
1742                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
1743                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
1744                                               0x80000000, 150, 2);
1745         if (ret) {
1746                 drm_err(&dev_priv->drm,
1747                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
1748                         ret, cdclk);
1749                 return;
1750         }
1751
1752         if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0) {
1753                 if (dev_priv->display.cdclk.hw.vco != vco)
1754                         adlp_cdclk_pll_crawl(dev_priv, vco);
1755         } else if (DISPLAY_VER(dev_priv) >= 11)
1756                 icl_cdclk_pll_update(dev_priv, vco);
1757         else
1758                 bxt_cdclk_pll_update(dev_priv, vco);
1759
1760         waveform = cdclk_squash_waveform(dev_priv, cdclk);
1761
1762         if (waveform)
1763                 clock = vco / 2;
1764         else
1765                 clock = cdclk;
1766
1767         if (HAS_CDCLK_SQUASH(dev_priv))
1768                 dg2_cdclk_squash_program(dev_priv, waveform);
1769
1770         val = bxt_cdclk_cd2x_div_sel(dev_priv, clock, vco) |
1771                 bxt_cdclk_cd2x_pipe(dev_priv, pipe) |
1772                 skl_cdclk_decimal(cdclk);
1773
1774         /*
1775          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1776          * enable otherwise.
1777          */
1778         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1779             cdclk >= 500000)
1780                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1781         intel_de_write(dev_priv, CDCLK_CTL, val);
1782
1783         if (pipe != INVALID_PIPE)
1784                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
1785
1786         if (DISPLAY_VER(dev_priv) >= 11) {
1787                 ret = snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1788                                       cdclk_config->voltage_level);
1789         } else {
1790                 /*
1791                  * The timeout isn't specified, the 2ms used here is based on
1792                  * experiment.
1793                  * FIXME: Waiting for the request completion could be delayed
1794                  * until the next PCODE request based on BSpec.
1795                  */
1796                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
1797                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
1798                                               cdclk_config->voltage_level,
1799                                               150, 2);
1800         }
1801
1802         if (ret) {
1803                 drm_err(&dev_priv->drm,
1804                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
1805                         ret, cdclk);
1806                 return;
1807         }
1808
1809         intel_update_cdclk(dev_priv);
1810
1811         if (DISPLAY_VER(dev_priv) >= 11)
1812                 /*
1813                  * Can't read out the voltage level :(
1814                  * Let's just assume everything is as expected.
1815                  */
1816                 dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level;
1817 }
1818
1819 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
1820 {
1821         u32 cdctl, expected;
1822         int cdclk, clock, vco;
1823
1824         intel_update_cdclk(dev_priv);
1825         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1826
1827         if (dev_priv->display.cdclk.hw.vco == 0 ||
1828             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1829                 goto sanitize;
1830
1831         /* DPLL okay; verify the cdclock
1832          *
1833          * Some BIOS versions leave an incorrect decimal frequency value and
1834          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
1835          * so sanitize this register.
1836          */
1837         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1838         /*
1839          * Let's ignore the pipe field, since BIOS could have configured the
1840          * dividers both synching to an active pipe, or asynchronously
1841          * (PIPE_NONE).
1842          */
1843         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
1844
1845         /* Make sure this is a legal cdclk value for the platform */
1846         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk);
1847         if (cdclk != dev_priv->display.cdclk.hw.cdclk)
1848                 goto sanitize;
1849
1850         /* Make sure the VCO is correct for the cdclk */
1851         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
1852         if (vco != dev_priv->display.cdclk.hw.vco)
1853                 goto sanitize;
1854
1855         expected = skl_cdclk_decimal(cdclk);
1856
1857         /* Figure out what CD2X divider we should be using for this cdclk */
1858         if (HAS_CDCLK_SQUASH(dev_priv))
1859                 clock = dev_priv->display.cdclk.hw.vco / 2;
1860         else
1861                 clock = dev_priv->display.cdclk.hw.cdclk;
1862
1863         expected |= bxt_cdclk_cd2x_div_sel(dev_priv, clock,
1864                                            dev_priv->display.cdclk.hw.vco);
1865
1866         /*
1867          * Disable SSA Precharge when CD clock frequency < 500 MHz,
1868          * enable otherwise.
1869          */
1870         if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
1871             dev_priv->display.cdclk.hw.cdclk >= 500000)
1872                 expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
1873
1874         if (cdctl == expected)
1875                 /* All well; nothing to sanitize */
1876                 return;
1877
1878 sanitize:
1879         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1880
1881         /* force cdclk programming */
1882         dev_priv->display.cdclk.hw.cdclk = 0;
1883
1884         /* force full PLL disable + enable */
1885         dev_priv->display.cdclk.hw.vco = -1;
1886 }
1887
1888 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
1889 {
1890         struct intel_cdclk_config cdclk_config;
1891
1892         bxt_sanitize_cdclk(dev_priv);
1893
1894         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1895             dev_priv->display.cdclk.hw.vco != 0)
1896                 return;
1897
1898         cdclk_config = dev_priv->display.cdclk.hw;
1899
1900         /*
1901          * FIXME:
1902          * - The initial CDCLK needs to be read from VBT.
1903          *   Need to make this change after VBT has changes for BXT.
1904          */
1905         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
1906         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
1907         cdclk_config.voltage_level =
1908                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1909
1910         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1911 }
1912
1913 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1914 {
1915         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1916
1917         cdclk_config.cdclk = cdclk_config.bypass;
1918         cdclk_config.vco = 0;
1919         cdclk_config.voltage_level =
1920                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
1921
1922         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1923 }
1924
1925 /**
1926  * intel_cdclk_init_hw - Initialize CDCLK hardware
1927  * @i915: i915 device
1928  *
1929  * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and
1930  * sanitizing the state of the hardware if needed. This is generally done only
1931  * during the display core initialization sequence, after which the DMC will
1932  * take care of turning CDCLK off/on as needed.
1933  */
1934 void intel_cdclk_init_hw(struct drm_i915_private *i915)
1935 {
1936         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1937                 bxt_cdclk_init_hw(i915);
1938         else if (DISPLAY_VER(i915) == 9)
1939                 skl_cdclk_init_hw(i915);
1940 }
1941
1942 /**
1943  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
1944  * @i915: i915 device
1945  *
1946  * Uninitialize CDCLK. This is done only during the display core
1947  * uninitialization sequence.
1948  */
1949 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
1950 {
1951         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
1952                 bxt_cdclk_uninit_hw(i915);
1953         else if (DISPLAY_VER(i915) == 9)
1954                 skl_cdclk_uninit_hw(i915);
1955 }
1956
1957 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
1958                                   const struct intel_cdclk_config *a,
1959                                   const struct intel_cdclk_config *b)
1960 {
1961         int a_div, b_div;
1962
1963         if (!HAS_CDCLK_CRAWL(dev_priv))
1964                 return false;
1965
1966         /*
1967          * The vco and cd2x divider will change independently
1968          * from each, so we disallow cd2x change when crawling.
1969          */
1970         a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
1971         b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
1972
1973         return a->vco != 0 && b->vco != 0 &&
1974                 a->vco != b->vco &&
1975                 a_div == b_div &&
1976                 a->ref == b->ref;
1977 }
1978
1979 static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
1980                                    const struct intel_cdclk_config *a,
1981                                    const struct intel_cdclk_config *b)
1982 {
1983         /*
1984          * FIXME should store a bit more state in intel_cdclk_config
1985          * to differentiate squasher vs. cd2x divider properly. For
1986          * the moment all platforms with squasher use a fixed cd2x
1987          * divider.
1988          */
1989         if (!HAS_CDCLK_SQUASH(dev_priv))
1990                 return false;
1991
1992         return a->cdclk != b->cdclk &&
1993                 a->vco != 0 &&
1994                 a->vco == b->vco &&
1995                 a->ref == b->ref;
1996 }
1997
1998 /**
1999  * intel_cdclk_needs_modeset - Determine if changong between the CDCLK
2000  *                             configurations requires a modeset on all pipes
2001  * @a: first CDCLK configuration
2002  * @b: second CDCLK configuration
2003  *
2004  * Returns:
2005  * True if changing between the two CDCLK configurations
2006  * requires all pipes to be off, false if not.
2007  */
2008 bool intel_cdclk_needs_modeset(const struct intel_cdclk_config *a,
2009                                const struct intel_cdclk_config *b)
2010 {
2011         return a->cdclk != b->cdclk ||
2012                 a->vco != b->vco ||
2013                 a->ref != b->ref;
2014 }
2015
2016 /**
2017  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
2018  *                               configurations requires only a cd2x divider update
2019  * @dev_priv: i915 device
2020  * @a: first CDCLK configuration
2021  * @b: second CDCLK configuration
2022  *
2023  * Returns:
2024  * True if changing between the two CDCLK configurations
2025  * can be done with just a cd2x divider update, false if not.
2026  */
2027 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
2028                                         const struct intel_cdclk_config *a,
2029                                         const struct intel_cdclk_config *b)
2030 {
2031         /* Older hw doesn't have the capability */
2032         if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
2033                 return false;
2034
2035         /*
2036          * FIXME should store a bit more state in intel_cdclk_config
2037          * to differentiate squasher vs. cd2x divider properly. For
2038          * the moment all platforms with squasher use a fixed cd2x
2039          * divider.
2040          */
2041         if (HAS_CDCLK_SQUASH(dev_priv))
2042                 return false;
2043
2044         return a->cdclk != b->cdclk &&
2045                 a->vco != 0 &&
2046                 a->vco == b->vco &&
2047                 a->ref == b->ref;
2048 }
2049
2050 /**
2051  * intel_cdclk_changed - Determine if two CDCLK configurations are different
2052  * @a: first CDCLK configuration
2053  * @b: second CDCLK configuration
2054  *
2055  * Returns:
2056  * True if the CDCLK configurations don't match, false if they do.
2057  */
2058 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
2059                                 const struct intel_cdclk_config *b)
2060 {
2061         return intel_cdclk_needs_modeset(a, b) ||
2062                 a->voltage_level != b->voltage_level;
2063 }
2064
2065 void intel_cdclk_dump_config(struct drm_i915_private *i915,
2066                              const struct intel_cdclk_config *cdclk_config,
2067                              const char *context)
2068 {
2069         drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
2070                     context, cdclk_config->cdclk, cdclk_config->vco,
2071                     cdclk_config->ref, cdclk_config->bypass,
2072                     cdclk_config->voltage_level);
2073 }
2074
2075 /**
2076  * intel_set_cdclk - Push the CDCLK configuration to the hardware
2077  * @dev_priv: i915 device
2078  * @cdclk_config: new CDCLK configuration
2079  * @pipe: pipe with which to synchronize the update
2080  *
2081  * Program the hardware based on the passed in CDCLK state,
2082  * if necessary.
2083  */
2084 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
2085                             const struct intel_cdclk_config *cdclk_config,
2086                             enum pipe pipe)
2087 {
2088         struct intel_encoder *encoder;
2089
2090         if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config))
2091                 return;
2092
2093         if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
2094                 return;
2095
2096         intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to");
2097
2098         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2099                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2100
2101                 intel_psr_pause(intel_dp);
2102         }
2103
2104         intel_audio_cdclk_change_pre(dev_priv);
2105
2106         /*
2107          * Lock aux/gmbus while we change cdclk in case those
2108          * functions use cdclk. Not all platforms/ports do,
2109          * but we'll lock them all for simplicity.
2110          */
2111         mutex_lock(&dev_priv->display.gmbus.mutex);
2112         for_each_intel_dp(&dev_priv->drm, encoder) {
2113                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2114
2115                 mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
2116                                      &dev_priv->display.gmbus.mutex);
2117         }
2118
2119         intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
2120
2121         for_each_intel_dp(&dev_priv->drm, encoder) {
2122                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2123
2124                 mutex_unlock(&intel_dp->aux.hw_mutex);
2125         }
2126         mutex_unlock(&dev_priv->display.gmbus.mutex);
2127
2128         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2129                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2130
2131                 intel_psr_resume(intel_dp);
2132         }
2133
2134         intel_audio_cdclk_change_post(dev_priv);
2135
2136         if (drm_WARN(&dev_priv->drm,
2137                      intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config),
2138                      "cdclk state doesn't match!\n")) {
2139                 intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]");
2140                 intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
2141         }
2142 }
2143
2144 /**
2145  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2146  * @state: intel atomic state
2147  *
2148  * Program the hardware before updating the HW plane state based on the
2149  * new CDCLK state, if necessary.
2150  */
2151 void
2152 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2153 {
2154         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2155         const struct intel_cdclk_state *old_cdclk_state =
2156                 intel_atomic_get_old_cdclk_state(state);
2157         const struct intel_cdclk_state *new_cdclk_state =
2158                 intel_atomic_get_new_cdclk_state(state);
2159         enum pipe pipe = new_cdclk_state->pipe;
2160
2161         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2162                                  &new_cdclk_state->actual))
2163                 return;
2164
2165         if (pipe == INVALID_PIPE ||
2166             old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) {
2167                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2168
2169                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2170         }
2171 }
2172
2173 /**
2174  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2175  * @state: intel atomic state
2176  *
2177  * Program the hardware after updating the HW plane state based on the
2178  * new CDCLK state, if necessary.
2179  */
2180 void
2181 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2182 {
2183         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2184         const struct intel_cdclk_state *old_cdclk_state =
2185                 intel_atomic_get_old_cdclk_state(state);
2186         const struct intel_cdclk_state *new_cdclk_state =
2187                 intel_atomic_get_new_cdclk_state(state);
2188         enum pipe pipe = new_cdclk_state->pipe;
2189
2190         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2191                                  &new_cdclk_state->actual))
2192                 return;
2193
2194         if (pipe != INVALID_PIPE &&
2195             old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) {
2196                 drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed);
2197
2198                 intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe);
2199         }
2200 }
2201
2202 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2203 {
2204         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2205         int pixel_rate = crtc_state->pixel_rate;
2206
2207         if (DISPLAY_VER(dev_priv) >= 10)
2208                 return DIV_ROUND_UP(pixel_rate, 2);
2209         else if (DISPLAY_VER(dev_priv) == 9 ||
2210                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2211                 return pixel_rate;
2212         else if (IS_CHERRYVIEW(dev_priv))
2213                 return DIV_ROUND_UP(pixel_rate * 100, 95);
2214         else if (crtc_state->double_wide)
2215                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2216         else
2217                 return DIV_ROUND_UP(pixel_rate * 100, 90);
2218 }
2219
2220 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2221 {
2222         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2223         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2224         struct intel_plane *plane;
2225         int min_cdclk = 0;
2226
2227         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2228                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2229
2230         return min_cdclk;
2231 }
2232
2233 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2234 {
2235         struct drm_i915_private *dev_priv =
2236                 to_i915(crtc_state->uapi.crtc->dev);
2237         int min_cdclk;
2238
2239         if (!crtc_state->hw.enable)
2240                 return 0;
2241
2242         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2243
2244         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2245         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2246                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2247
2248         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2249          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2250          * there may be audio corruption or screen corruption." This cdclk
2251          * restriction for GLK is 316.8 MHz.
2252          */
2253         if (intel_crtc_has_dp_encoder(crtc_state) &&
2254             crtc_state->has_audio &&
2255             crtc_state->port_clock >= 540000 &&
2256             crtc_state->lane_count == 4) {
2257                 if (DISPLAY_VER(dev_priv) == 10) {
2258                         /* Display WA #1145: glk */
2259                         min_cdclk = max(316800, min_cdclk);
2260                 } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2261                         /* Display WA #1144: skl,bxt */
2262                         min_cdclk = max(432000, min_cdclk);
2263                 }
2264         }
2265
2266         /*
2267          * According to BSpec, "The CD clock frequency must be at least twice
2268          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2269          */
2270         if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2271                 min_cdclk = max(2 * 96000, min_cdclk);
2272
2273         /*
2274          * "For DP audio configuration, cdclk frequency shall be set to
2275          *  meet the following requirements:
2276          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2277          *  270                    | 320 or higher
2278          *  162                    | 200 or higher"
2279          */
2280         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2281             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2282                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2283
2284         /*
2285          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2286          * than 320000KHz.
2287          */
2288         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2289             IS_VALLEYVIEW(dev_priv))
2290                 min_cdclk = max(320000, min_cdclk);
2291
2292         /*
2293          * On Geminilake once the CDCLK gets as low as 79200
2294          * picture gets unstable, despite that values are
2295          * correct for DSI PLL and DE PLL.
2296          */
2297         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2298             IS_GEMINILAKE(dev_priv))
2299                 min_cdclk = max(158400, min_cdclk);
2300
2301         /* Account for additional needs from the planes */
2302         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2303
2304         /*
2305          * When we decide to use only one VDSC engine, since
2306          * each VDSC operates with 1 ppc throughput, pixel clock
2307          * cannot be higher than the VDSC clock (cdclk)
2308          */
2309         if (crtc_state->dsc.compression_enable && !crtc_state->dsc.dsc_split)
2310                 min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate);
2311
2312         /*
2313          * HACK. Currently for TGL/DG2 platforms we calculate
2314          * min_cdclk initially based on pixel_rate divided
2315          * by 2, accounting for also plane requirements,
2316          * however in some cases the lowest possible CDCLK
2317          * doesn't work and causing the underruns.
2318          * Explicitly stating here that this seems to be currently
2319          * rather a Hack, than final solution.
2320          */
2321         if (IS_TIGERLAKE(dev_priv) || IS_DG2(dev_priv)) {
2322                 /*
2323                  * Clamp to max_cdclk_freq in case pixel rate is higher,
2324                  * in order not to break an 8K, but still leave W/A at place.
2325                  */
2326                 min_cdclk = max_t(int, min_cdclk,
2327                                   min_t(int, crtc_state->pixel_rate,
2328                                         dev_priv->display.cdclk.max_cdclk_freq));
2329         }
2330
2331         return min_cdclk;
2332 }
2333
2334 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2335 {
2336         struct intel_atomic_state *state = cdclk_state->base.state;
2337         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2338         const struct intel_bw_state *bw_state;
2339         struct intel_crtc *crtc;
2340         struct intel_crtc_state *crtc_state;
2341         int min_cdclk, i;
2342         enum pipe pipe;
2343
2344         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2345                 int ret;
2346
2347                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2348                 if (min_cdclk < 0)
2349                         return min_cdclk;
2350
2351                 if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2352                         continue;
2353
2354                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2355
2356                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2357                 if (ret)
2358                         return ret;
2359         }
2360
2361         bw_state = intel_atomic_get_new_bw_state(state);
2362         if (bw_state) {
2363                 min_cdclk = intel_bw_min_cdclk(dev_priv, bw_state);
2364
2365                 if (cdclk_state->bw_min_cdclk != min_cdclk) {
2366                         int ret;
2367
2368                         cdclk_state->bw_min_cdclk = min_cdclk;
2369
2370                         ret = intel_atomic_lock_global_state(&cdclk_state->base);
2371                         if (ret)
2372                                 return ret;
2373                 }
2374         }
2375
2376         min_cdclk = max(cdclk_state->force_min_cdclk,
2377                         cdclk_state->bw_min_cdclk);
2378         for_each_pipe(dev_priv, pipe)
2379                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2380
2381         if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) {
2382                 drm_dbg_kms(&dev_priv->drm,
2383                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2384                             min_cdclk, dev_priv->display.cdclk.max_cdclk_freq);
2385                 return -EINVAL;
2386         }
2387
2388         return min_cdclk;
2389 }
2390
2391 /*
2392  * Account for port clock min voltage level requirements.
2393  * This only really does something on DISPLA_VER >= 11 but can be
2394  * called on earlier platforms as well.
2395  *
2396  * Note that this functions assumes that 0 is
2397  * the lowest voltage value, and higher values
2398  * correspond to increasingly higher voltages.
2399  *
2400  * Should that relationship no longer hold on
2401  * future platforms this code will need to be
2402  * adjusted.
2403  */
2404 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2405 {
2406         struct intel_atomic_state *state = cdclk_state->base.state;
2407         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2408         struct intel_crtc *crtc;
2409         struct intel_crtc_state *crtc_state;
2410         u8 min_voltage_level;
2411         int i;
2412         enum pipe pipe;
2413
2414         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2415                 int ret;
2416
2417                 if (crtc_state->hw.enable)
2418                         min_voltage_level = crtc_state->min_voltage_level;
2419                 else
2420                         min_voltage_level = 0;
2421
2422                 if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2423                         continue;
2424
2425                 cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2426
2427                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2428                 if (ret)
2429                         return ret;
2430         }
2431
2432         min_voltage_level = 0;
2433         for_each_pipe(dev_priv, pipe)
2434                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2435                                         min_voltage_level);
2436
2437         return min_voltage_level;
2438 }
2439
2440 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2441 {
2442         struct intel_atomic_state *state = cdclk_state->base.state;
2443         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2444         int min_cdclk, cdclk;
2445
2446         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2447         if (min_cdclk < 0)
2448                 return min_cdclk;
2449
2450         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2451
2452         cdclk_state->logical.cdclk = cdclk;
2453         cdclk_state->logical.voltage_level =
2454                 vlv_calc_voltage_level(dev_priv, cdclk);
2455
2456         if (!cdclk_state->active_pipes) {
2457                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2458
2459                 cdclk_state->actual.cdclk = cdclk;
2460                 cdclk_state->actual.voltage_level =
2461                         vlv_calc_voltage_level(dev_priv, cdclk);
2462         } else {
2463                 cdclk_state->actual = cdclk_state->logical;
2464         }
2465
2466         return 0;
2467 }
2468
2469 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2470 {
2471         int min_cdclk, cdclk;
2472
2473         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2474         if (min_cdclk < 0)
2475                 return min_cdclk;
2476
2477         cdclk = bdw_calc_cdclk(min_cdclk);
2478
2479         cdclk_state->logical.cdclk = cdclk;
2480         cdclk_state->logical.voltage_level =
2481                 bdw_calc_voltage_level(cdclk);
2482
2483         if (!cdclk_state->active_pipes) {
2484                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2485
2486                 cdclk_state->actual.cdclk = cdclk;
2487                 cdclk_state->actual.voltage_level =
2488                         bdw_calc_voltage_level(cdclk);
2489         } else {
2490                 cdclk_state->actual = cdclk_state->logical;
2491         }
2492
2493         return 0;
2494 }
2495
2496 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
2497 {
2498         struct intel_atomic_state *state = cdclk_state->base.state;
2499         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2500         struct intel_crtc *crtc;
2501         struct intel_crtc_state *crtc_state;
2502         int vco, i;
2503
2504         vco = cdclk_state->logical.vco;
2505         if (!vco)
2506                 vco = dev_priv->skl_preferred_vco_freq;
2507
2508         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2509                 if (!crtc_state->hw.enable)
2510                         continue;
2511
2512                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
2513                         continue;
2514
2515                 /*
2516                  * DPLL0 VCO may need to be adjusted to get the correct
2517                  * clock for eDP. This will affect cdclk as well.
2518                  */
2519                 switch (crtc_state->port_clock / 2) {
2520                 case 108000:
2521                 case 216000:
2522                         vco = 8640000;
2523                         break;
2524                 default:
2525                         vco = 8100000;
2526                         break;
2527                 }
2528         }
2529
2530         return vco;
2531 }
2532
2533 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2534 {
2535         int min_cdclk, cdclk, vco;
2536
2537         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2538         if (min_cdclk < 0)
2539                 return min_cdclk;
2540
2541         vco = skl_dpll0_vco(cdclk_state);
2542
2543         cdclk = skl_calc_cdclk(min_cdclk, vco);
2544
2545         cdclk_state->logical.vco = vco;
2546         cdclk_state->logical.cdclk = cdclk;
2547         cdclk_state->logical.voltage_level =
2548                 skl_calc_voltage_level(cdclk);
2549
2550         if (!cdclk_state->active_pipes) {
2551                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
2552
2553                 cdclk_state->actual.vco = vco;
2554                 cdclk_state->actual.cdclk = cdclk;
2555                 cdclk_state->actual.voltage_level =
2556                         skl_calc_voltage_level(cdclk);
2557         } else {
2558                 cdclk_state->actual = cdclk_state->logical;
2559         }
2560
2561         return 0;
2562 }
2563
2564 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2565 {
2566         struct intel_atomic_state *state = cdclk_state->base.state;
2567         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2568         int min_cdclk, min_voltage_level, cdclk, vco;
2569
2570         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2571         if (min_cdclk < 0)
2572                 return min_cdclk;
2573
2574         min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
2575         if (min_voltage_level < 0)
2576                 return min_voltage_level;
2577
2578         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
2579         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2580
2581         cdclk_state->logical.vco = vco;
2582         cdclk_state->logical.cdclk = cdclk;
2583         cdclk_state->logical.voltage_level =
2584                 max_t(int, min_voltage_level,
2585                       intel_cdclk_calc_voltage_level(dev_priv, cdclk));
2586
2587         if (!cdclk_state->active_pipes) {
2588                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2589                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2590
2591                 cdclk_state->actual.vco = vco;
2592                 cdclk_state->actual.cdclk = cdclk;
2593                 cdclk_state->actual.voltage_level =
2594                         intel_cdclk_calc_voltage_level(dev_priv, cdclk);
2595         } else {
2596                 cdclk_state->actual = cdclk_state->logical;
2597         }
2598
2599         return 0;
2600 }
2601
2602 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2603 {
2604         int min_cdclk;
2605
2606         /*
2607          * We can't change the cdclk frequency, but we still want to
2608          * check that the required minimum frequency doesn't exceed
2609          * the actual cdclk frequency.
2610          */
2611         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2612         if (min_cdclk < 0)
2613                 return min_cdclk;
2614
2615         return 0;
2616 }
2617
2618 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
2619 {
2620         struct intel_cdclk_state *cdclk_state;
2621
2622         cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
2623         if (!cdclk_state)
2624                 return NULL;
2625
2626         cdclk_state->pipe = INVALID_PIPE;
2627
2628         return &cdclk_state->base;
2629 }
2630
2631 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
2632                                       struct intel_global_state *state)
2633 {
2634         kfree(state);
2635 }
2636
2637 static const struct intel_global_state_funcs intel_cdclk_funcs = {
2638         .atomic_duplicate_state = intel_cdclk_duplicate_state,
2639         .atomic_destroy_state = intel_cdclk_destroy_state,
2640 };
2641
2642 struct intel_cdclk_state *
2643 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
2644 {
2645         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2646         struct intel_global_state *cdclk_state;
2647
2648         cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj);
2649         if (IS_ERR(cdclk_state))
2650                 return ERR_CAST(cdclk_state);
2651
2652         return to_intel_cdclk_state(cdclk_state);
2653 }
2654
2655 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
2656                              bool *need_cdclk_calc)
2657 {
2658         const struct intel_cdclk_state *old_cdclk_state;
2659         const struct intel_cdclk_state *new_cdclk_state;
2660         struct intel_plane_state *plane_state;
2661         struct intel_plane *plane;
2662         int ret;
2663         int i;
2664
2665         /*
2666          * active_planes bitmask has been updated, and potentially affected
2667          * planes are part of the state. We can now compute the minimum cdclk
2668          * for each plane.
2669          */
2670         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2671                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
2672                 if (ret)
2673                         return ret;
2674         }
2675
2676         ret = intel_bw_calc_min_cdclk(state, need_cdclk_calc);
2677         if (ret)
2678                 return ret;
2679
2680         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2681         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
2682
2683         if (new_cdclk_state &&
2684             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
2685                 *need_cdclk_calc = true;
2686
2687         return 0;
2688 }
2689
2690 int intel_cdclk_init(struct drm_i915_private *dev_priv)
2691 {
2692         struct intel_cdclk_state *cdclk_state;
2693
2694         cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
2695         if (!cdclk_state)
2696                 return -ENOMEM;
2697
2698         intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj,
2699                                      &cdclk_state->base, &intel_cdclk_funcs);
2700
2701         return 0;
2702 }
2703
2704 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
2705 {
2706         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2707         const struct intel_cdclk_state *old_cdclk_state;
2708         struct intel_cdclk_state *new_cdclk_state;
2709         enum pipe pipe = INVALID_PIPE;
2710         int ret;
2711
2712         new_cdclk_state = intel_atomic_get_cdclk_state(state);
2713         if (IS_ERR(new_cdclk_state))
2714                 return PTR_ERR(new_cdclk_state);
2715
2716         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
2717
2718         new_cdclk_state->active_pipes =
2719                 intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
2720
2721         ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
2722         if (ret)
2723                 return ret;
2724
2725         if (intel_cdclk_changed(&old_cdclk_state->actual,
2726                                 &new_cdclk_state->actual)) {
2727                 /*
2728                  * Also serialize commits across all crtcs
2729                  * if the actual hw needs to be poked.
2730                  */
2731                 ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
2732                 if (ret)
2733                         return ret;
2734         } else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
2735                    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
2736                    intel_cdclk_changed(&old_cdclk_state->logical,
2737                                        &new_cdclk_state->logical)) {
2738                 ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
2739                 if (ret)
2740                         return ret;
2741         } else {
2742                 return 0;
2743         }
2744
2745         if (is_power_of_2(new_cdclk_state->active_pipes) &&
2746             intel_cdclk_can_cd2x_update(dev_priv,
2747                                         &old_cdclk_state->actual,
2748                                         &new_cdclk_state->actual)) {
2749                 struct intel_crtc *crtc;
2750                 struct intel_crtc_state *crtc_state;
2751
2752                 pipe = ilog2(new_cdclk_state->active_pipes);
2753                 crtc = intel_crtc_for_pipe(dev_priv, pipe);
2754
2755                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
2756                 if (IS_ERR(crtc_state))
2757                         return PTR_ERR(crtc_state);
2758
2759                 if (intel_crtc_needs_modeset(crtc_state))
2760                         pipe = INVALID_PIPE;
2761         }
2762
2763         if (intel_cdclk_can_squash(dev_priv,
2764                                    &old_cdclk_state->actual,
2765                                    &new_cdclk_state->actual)) {
2766                 drm_dbg_kms(&dev_priv->drm,
2767                             "Can change cdclk via squashing\n");
2768         } else if (intel_cdclk_can_crawl(dev_priv,
2769                                          &old_cdclk_state->actual,
2770                                          &new_cdclk_state->actual)) {
2771                 drm_dbg_kms(&dev_priv->drm,
2772                             "Can change cdclk via crawling\n");
2773         } else if (pipe != INVALID_PIPE) {
2774                 new_cdclk_state->pipe = pipe;
2775
2776                 drm_dbg_kms(&dev_priv->drm,
2777                             "Can change cdclk cd2x divider with pipe %c active\n",
2778                             pipe_name(pipe));
2779         } else if (intel_cdclk_needs_modeset(&old_cdclk_state->actual,
2780                                              &new_cdclk_state->actual)) {
2781                 /* All pipes must be switched off while we change the cdclk. */
2782                 ret = intel_modeset_all_pipes(state, "CDCLK change");
2783                 if (ret)
2784                         return ret;
2785
2786                 drm_dbg_kms(&dev_priv->drm,
2787                             "Modeset required for cdclk change\n");
2788         }
2789
2790         drm_dbg_kms(&dev_priv->drm,
2791                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
2792                     new_cdclk_state->logical.cdclk,
2793                     new_cdclk_state->actual.cdclk);
2794         drm_dbg_kms(&dev_priv->drm,
2795                     "New voltage level calculated to be logical %u, actual %u\n",
2796                     new_cdclk_state->logical.voltage_level,
2797                     new_cdclk_state->actual.voltage_level);
2798
2799         return 0;
2800 }
2801
2802 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
2803 {
2804         int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq;
2805
2806         if (DISPLAY_VER(dev_priv) >= 10)
2807                 return 2 * max_cdclk_freq;
2808         else if (DISPLAY_VER(dev_priv) == 9 ||
2809                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2810                 return max_cdclk_freq;
2811         else if (IS_CHERRYVIEW(dev_priv))
2812                 return max_cdclk_freq*95/100;
2813         else if (DISPLAY_VER(dev_priv) < 4)
2814                 return 2*max_cdclk_freq*90/100;
2815         else
2816                 return max_cdclk_freq*90/100;
2817 }
2818
2819 /**
2820  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
2821  * @dev_priv: i915 device
2822  *
2823  * Determine the maximum CDCLK frequency the platform supports, and also
2824  * derive the maximum dot clock frequency the maximum CDCLK frequency
2825  * allows.
2826  */
2827 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
2828 {
2829         if (IS_JSL_EHL(dev_priv)) {
2830                 if (dev_priv->display.cdclk.hw.ref == 24000)
2831                         dev_priv->display.cdclk.max_cdclk_freq = 552000;
2832                 else
2833                         dev_priv->display.cdclk.max_cdclk_freq = 556800;
2834         } else if (DISPLAY_VER(dev_priv) >= 11) {
2835                 if (dev_priv->display.cdclk.hw.ref == 24000)
2836                         dev_priv->display.cdclk.max_cdclk_freq = 648000;
2837                 else
2838                         dev_priv->display.cdclk.max_cdclk_freq = 652800;
2839         } else if (IS_GEMINILAKE(dev_priv)) {
2840                 dev_priv->display.cdclk.max_cdclk_freq = 316800;
2841         } else if (IS_BROXTON(dev_priv)) {
2842                 dev_priv->display.cdclk.max_cdclk_freq = 624000;
2843         } else if (DISPLAY_VER(dev_priv) == 9) {
2844                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
2845                 int max_cdclk, vco;
2846
2847                 vco = dev_priv->skl_preferred_vco_freq;
2848                 drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
2849
2850                 /*
2851                  * Use the lower (vco 8640) cdclk values as a
2852                  * first guess. skl_calc_cdclk() will correct it
2853                  * if the preferred vco is 8100 instead.
2854                  */
2855                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
2856                         max_cdclk = 617143;
2857                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
2858                         max_cdclk = 540000;
2859                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
2860                         max_cdclk = 432000;
2861                 else
2862                         max_cdclk = 308571;
2863
2864                 dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
2865         } else if (IS_BROADWELL(dev_priv))  {
2866                 /*
2867                  * FIXME with extra cooling we can allow
2868                  * 540 MHz for ULX and 675 Mhz for ULT.
2869                  * How can we know if extra cooling is
2870                  * available? PCI ID, VTB, something else?
2871                  */
2872                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
2873                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
2874                 else if (IS_BDW_ULX(dev_priv))
2875                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
2876                 else if (IS_BDW_ULT(dev_priv))
2877                         dev_priv->display.cdclk.max_cdclk_freq = 540000;
2878                 else
2879                         dev_priv->display.cdclk.max_cdclk_freq = 675000;
2880         } else if (IS_CHERRYVIEW(dev_priv)) {
2881                 dev_priv->display.cdclk.max_cdclk_freq = 320000;
2882         } else if (IS_VALLEYVIEW(dev_priv)) {
2883                 dev_priv->display.cdclk.max_cdclk_freq = 400000;
2884         } else {
2885                 /* otherwise assume cdclk is fixed */
2886                 dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
2887         }
2888
2889         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
2890
2891         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
2892                 dev_priv->display.cdclk.max_cdclk_freq);
2893
2894         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
2895                 dev_priv->max_dotclk_freq);
2896 }
2897
2898 /**
2899  * intel_update_cdclk - Determine the current CDCLK frequency
2900  * @dev_priv: i915 device
2901  *
2902  * Determine the current CDCLK frequency.
2903  */
2904 void intel_update_cdclk(struct drm_i915_private *dev_priv)
2905 {
2906         intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw);
2907
2908         /*
2909          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
2910          * Programmng [sic] note: bit[9:2] should be programmed to the number
2911          * of cdclk that generates 4MHz reference clock freq which is used to
2912          * generate GMBus clock. This will vary with the cdclk freq.
2913          */
2914         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
2915                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
2916                                DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000));
2917 }
2918
2919 static int dg1_rawclk(struct drm_i915_private *dev_priv)
2920 {
2921         /*
2922          * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
2923          * "Program Numerator=2, Denominator=4, Divider=37 decimal."
2924          */
2925         intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
2926                        CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
2927
2928         return 38400;
2929 }
2930
2931 static int cnp_rawclk(struct drm_i915_private *dev_priv)
2932 {
2933         u32 rawclk;
2934         int divider, fraction;
2935
2936         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
2937                 /* 24 MHz */
2938                 divider = 24000;
2939                 fraction = 0;
2940         } else {
2941                 /* 19.2 MHz */
2942                 divider = 19000;
2943                 fraction = 200;
2944         }
2945
2946         rawclk = CNP_RAWCLK_DIV(divider / 1000);
2947         if (fraction) {
2948                 int numerator = 1;
2949
2950                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
2951                                                            fraction) - 1);
2952                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
2953                         rawclk |= ICP_RAWCLK_NUM(numerator);
2954         }
2955
2956         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
2957         return divider + fraction;
2958 }
2959
2960 static int pch_rawclk(struct drm_i915_private *dev_priv)
2961 {
2962         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
2963 }
2964
2965 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
2966 {
2967         /* RAWCLK_FREQ_VLV register updated from power well code */
2968         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
2969                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
2970 }
2971
2972 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
2973 {
2974         u32 clkcfg;
2975
2976         /*
2977          * hrawclock is 1/4 the FSB frequency
2978          *
2979          * Note that this only reads the state of the FSB
2980          * straps, not the actual FSB frequency. Some BIOSen
2981          * let you configure each independently. Ideally we'd
2982          * read out the actual FSB frequency but sadly we
2983          * don't know which registers have that information,
2984          * and all the relevant docs have gone to bit heaven :(
2985          */
2986         clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
2987
2988         if (IS_MOBILE(dev_priv)) {
2989                 switch (clkcfg) {
2990                 case CLKCFG_FSB_400:
2991                         return 100000;
2992                 case CLKCFG_FSB_533:
2993                         return 133333;
2994                 case CLKCFG_FSB_667:
2995                         return 166667;
2996                 case CLKCFG_FSB_800:
2997                         return 200000;
2998                 case CLKCFG_FSB_1067:
2999                         return 266667;
3000                 case CLKCFG_FSB_1333:
3001                         return 333333;
3002                 default:
3003                         MISSING_CASE(clkcfg);
3004                         return 133333;
3005                 }
3006         } else {
3007                 switch (clkcfg) {
3008                 case CLKCFG_FSB_400_ALT:
3009                         return 100000;
3010                 case CLKCFG_FSB_533:
3011                         return 133333;
3012                 case CLKCFG_FSB_667:
3013                         return 166667;
3014                 case CLKCFG_FSB_800:
3015                         return 200000;
3016                 case CLKCFG_FSB_1067_ALT:
3017                         return 266667;
3018                 case CLKCFG_FSB_1333_ALT:
3019                         return 333333;
3020                 case CLKCFG_FSB_1600_ALT:
3021                         return 400000;
3022                 default:
3023                         return 133333;
3024                 }
3025         }
3026 }
3027
3028 /**
3029  * intel_read_rawclk - Determine the current RAWCLK frequency
3030  * @dev_priv: i915 device
3031  *
3032  * Determine the current RAWCLK frequency. RAWCLK is a fixed
3033  * frequency clock so this needs to done only once.
3034  */
3035 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
3036 {
3037         u32 freq;
3038
3039         if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
3040                 freq = dg1_rawclk(dev_priv);
3041         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTP)
3042                 /*
3043                  * MTL always uses a 38.4 MHz rawclk.  The bspec tells us
3044                  * "RAWCLK_FREQ defaults to the values for 38.4 and does
3045                  * not need to be programmed."
3046                  */
3047                 freq = 38400;
3048         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3049                 freq = cnp_rawclk(dev_priv);
3050         else if (HAS_PCH_SPLIT(dev_priv))
3051                 freq = pch_rawclk(dev_priv);
3052         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3053                 freq = vlv_hrawclk(dev_priv);
3054         else if (DISPLAY_VER(dev_priv) >= 3)
3055                 freq = i9xx_hrawclk(dev_priv);
3056         else
3057                 /* no rawclk on other platforms, or no need to know it */
3058                 return 0;
3059
3060         return freq;
3061 }
3062
3063 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
3064         .get_cdclk = bxt_get_cdclk,
3065         .set_cdclk = bxt_set_cdclk,
3066         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3067         .calc_voltage_level = tgl_calc_voltage_level,
3068 };
3069
3070 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
3071         .get_cdclk = bxt_get_cdclk,
3072         .set_cdclk = bxt_set_cdclk,
3073         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3074         .calc_voltage_level = ehl_calc_voltage_level,
3075 };
3076
3077 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
3078         .get_cdclk = bxt_get_cdclk,
3079         .set_cdclk = bxt_set_cdclk,
3080         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3081         .calc_voltage_level = icl_calc_voltage_level,
3082 };
3083
3084 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
3085         .get_cdclk = bxt_get_cdclk,
3086         .set_cdclk = bxt_set_cdclk,
3087         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3088         .calc_voltage_level = bxt_calc_voltage_level,
3089 };
3090
3091 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
3092         .get_cdclk = skl_get_cdclk,
3093         .set_cdclk = skl_set_cdclk,
3094         .modeset_calc_cdclk = skl_modeset_calc_cdclk,
3095 };
3096
3097 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
3098         .get_cdclk = bdw_get_cdclk,
3099         .set_cdclk = bdw_set_cdclk,
3100         .modeset_calc_cdclk = bdw_modeset_calc_cdclk,
3101 };
3102
3103 static const struct intel_cdclk_funcs chv_cdclk_funcs = {
3104         .get_cdclk = vlv_get_cdclk,
3105         .set_cdclk = chv_set_cdclk,
3106         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3107 };
3108
3109 static const struct intel_cdclk_funcs vlv_cdclk_funcs = {
3110         .get_cdclk = vlv_get_cdclk,
3111         .set_cdclk = vlv_set_cdclk,
3112         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3113 };
3114
3115 static const struct intel_cdclk_funcs hsw_cdclk_funcs = {
3116         .get_cdclk = hsw_get_cdclk,
3117         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3118 };
3119
3120 /* SNB, IVB, 965G, 945G */
3121 static const struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
3122         .get_cdclk = fixed_400mhz_get_cdclk,
3123         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3124 };
3125
3126 static const struct intel_cdclk_funcs ilk_cdclk_funcs = {
3127         .get_cdclk = fixed_450mhz_get_cdclk,
3128         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3129 };
3130
3131 static const struct intel_cdclk_funcs gm45_cdclk_funcs = {
3132         .get_cdclk = gm45_get_cdclk,
3133         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3134 };
3135
3136 /* G45 uses G33 */
3137
3138 static const struct intel_cdclk_funcs i965gm_cdclk_funcs = {
3139         .get_cdclk = i965gm_get_cdclk,
3140         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3141 };
3142
3143 /* i965G uses fixed 400 */
3144
3145 static const struct intel_cdclk_funcs pnv_cdclk_funcs = {
3146         .get_cdclk = pnv_get_cdclk,
3147         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3148 };
3149
3150 static const struct intel_cdclk_funcs g33_cdclk_funcs = {
3151         .get_cdclk = g33_get_cdclk,
3152         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3153 };
3154
3155 static const struct intel_cdclk_funcs i945gm_cdclk_funcs = {
3156         .get_cdclk = i945gm_get_cdclk,
3157         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3158 };
3159
3160 /* i945G uses fixed 400 */
3161
3162 static const struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3163         .get_cdclk = i915gm_get_cdclk,
3164         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3165 };
3166
3167 static const struct intel_cdclk_funcs i915g_cdclk_funcs = {
3168         .get_cdclk = fixed_333mhz_get_cdclk,
3169         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3170 };
3171
3172 static const struct intel_cdclk_funcs i865g_cdclk_funcs = {
3173         .get_cdclk = fixed_266mhz_get_cdclk,
3174         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3175 };
3176
3177 static const struct intel_cdclk_funcs i85x_cdclk_funcs = {
3178         .get_cdclk = i85x_get_cdclk,
3179         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3180 };
3181
3182 static const struct intel_cdclk_funcs i845g_cdclk_funcs = {
3183         .get_cdclk = fixed_200mhz_get_cdclk,
3184         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3185 };
3186
3187 static const struct intel_cdclk_funcs i830_cdclk_funcs = {
3188         .get_cdclk = fixed_133mhz_get_cdclk,
3189         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3190 };
3191
3192 /**
3193  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3194  * @dev_priv: i915 device
3195  */
3196 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3197 {
3198         if (IS_DG2(dev_priv)) {
3199                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3200                 dev_priv->display.cdclk.table = dg2_cdclk_table;
3201         } else if (IS_ALDERLAKE_P(dev_priv)) {
3202                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3203                 /* Wa_22011320316:adl-p[a0] */
3204                 if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
3205                         dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
3206                 else
3207                         dev_priv->display.cdclk.table = adlp_cdclk_table;
3208         } else if (IS_ROCKETLAKE(dev_priv)) {
3209                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3210                 dev_priv->display.cdclk.table = rkl_cdclk_table;
3211         } else if (DISPLAY_VER(dev_priv) >= 12) {
3212                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3213                 dev_priv->display.cdclk.table = icl_cdclk_table;
3214         } else if (IS_JSL_EHL(dev_priv)) {
3215                 dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs;
3216                 dev_priv->display.cdclk.table = icl_cdclk_table;
3217         } else if (DISPLAY_VER(dev_priv) >= 11) {
3218                 dev_priv->display.funcs.cdclk = &icl_cdclk_funcs;
3219                 dev_priv->display.cdclk.table = icl_cdclk_table;
3220         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3221                 dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs;
3222                 if (IS_GEMINILAKE(dev_priv))
3223                         dev_priv->display.cdclk.table = glk_cdclk_table;
3224                 else
3225                         dev_priv->display.cdclk.table = bxt_cdclk_table;
3226         } else if (DISPLAY_VER(dev_priv) == 9) {
3227                 dev_priv->display.funcs.cdclk = &skl_cdclk_funcs;
3228         } else if (IS_BROADWELL(dev_priv)) {
3229                 dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs;
3230         } else if (IS_HASWELL(dev_priv)) {
3231                 dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs;
3232         } else if (IS_CHERRYVIEW(dev_priv)) {
3233                 dev_priv->display.funcs.cdclk = &chv_cdclk_funcs;
3234         } else if (IS_VALLEYVIEW(dev_priv)) {
3235                 dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs;
3236         } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3237                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3238         } else if (IS_IRONLAKE(dev_priv)) {
3239                 dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs;
3240         } else if (IS_GM45(dev_priv)) {
3241                 dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs;
3242         } else if (IS_G45(dev_priv)) {
3243                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3244         } else if (IS_I965GM(dev_priv)) {
3245                 dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs;
3246         } else if (IS_I965G(dev_priv)) {
3247                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3248         } else if (IS_PINEVIEW(dev_priv)) {
3249                 dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs;
3250         } else if (IS_G33(dev_priv)) {
3251                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3252         } else if (IS_I945GM(dev_priv)) {
3253                 dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs;
3254         } else if (IS_I945G(dev_priv)) {
3255                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3256         } else if (IS_I915GM(dev_priv)) {
3257                 dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs;
3258         } else if (IS_I915G(dev_priv)) {
3259                 dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs;
3260         } else if (IS_I865G(dev_priv)) {
3261                 dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs;
3262         } else if (IS_I85X(dev_priv)) {
3263                 dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs;
3264         } else if (IS_I845G(dev_priv)) {
3265                 dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs;
3266         } else if (IS_I830(dev_priv)) {
3267                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3268         }
3269
3270         if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk,
3271                      "Unknown platform. Assuming i830\n"))
3272                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3273 }