drivers/gpu/drm/i915/display/intel_cdclk.c

   1 /*
   2  * Copyright © 2006-2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #include <linux/time.h>
  25
  26 #include "hsw_ips.h"
  27 #include "i915_reg.h"
  28 #include "intel_atomic.h"
  29 #include "intel_atomic_plane.h"
  30 #include "intel_audio.h"
  31 #include "intel_bw.h"
  32 #include "intel_cdclk.h"
  33 #include "intel_crtc.h"
  34 #include "intel_de.h"
  35 #include "intel_dp.h"
  36 #include "intel_display_types.h"
  37 #include "intel_mchbar_regs.h"
  38 #include "intel_pci_config.h"
  39 #include "intel_pcode.h"
  40 #include "intel_psr.h"
  41 #include "intel_vdsc.h"
  42 #include "skl_watermark.h"
  43 #include "skl_watermark_regs.h"
  44 #include "vlv_sideband.h"
  45
  46 /**
  47  * DOC: CDCLK / RAWCLK
  48  *
  49  * The display engine uses several different clocks to do its work. There
  50  * are two main clocks involved that aren't directly related to the actual
  51  * pixel clock or any symbol/bit clock of the actual output port. These
  52  * are the core display clock (CDCLK) and RAWCLK.
  53  *
  54  * CDCLK clocks most of the display pipe logic, and thus its frequency
  55  * must be high enough to support the rate at which pixels are flowing
  56  * through the pipes. Downscaling must also be accounted as that increases
  57  * the effective pixel rate.
  58  *
  59  * On several platforms the CDCLK frequency can be changed dynamically
  60  * to minimize power consumption for a given display configuration.
  61  * Typically changes to the CDCLK frequency require all the display pipes
  62  * to be shut down while the frequency is being changed.
  63  *
  64  * On SKL+ the DMC will toggle the CDCLK off/on during DC5/6 entry/exit.
  65  * DMC will not change the active CDCLK frequency however, so that part
  66  * will still be performed by the driver directly.
  67  *
  68  * There are multiple components involved in the generation of the CDCLK
  69  * frequency:
  70  *
  71  * - We have the CDCLK PLL, which generates an output clock based on a
  72  *   reference clock and a ratio parameter.
  73  * - The CD2X Divider, which divides the output of the PLL based on a
  74  *   divisor selected from a set of pre-defined choices.
  75  * - The CD2X Squasher, which further divides the output based on a
  76  *   waveform represented as a sequence of bits where each zero
  77  *   "squashes out" a clock cycle.
  78  * - And, finally, a fixed divider that divides the output frequency by 2.
  79  *
  80  * As such, the resulting CDCLK frequency can be calculated with the
  81  * following formula:
  82  *
  83  *     cdclk = vco / cd2x_div / (sq_len / sq_div) / 2
  84  *
  85  * , where vco is the frequency generated by the PLL; cd2x_div
  86  * represents the CD2X Divider; sq_len and sq_div are the bit length
  87  * and the number of high bits for the CD2X Squasher waveform, respectively;
  88  * and 2 represents the fixed divider.
  89  *
  90  * Note that some older platforms do not contain the CD2X Divider
  91  * and/or CD2X Squasher, in which case we can ignore their respective
  92  * factors in the formula above.
  93  *
  94  * Several methods exist to change the CDCLK frequency, which ones are
  95  * supported depends on the platform:
  96  *
  97  * - Full PLL disable + re-enable with new VCO frequency. Pipes must be inactive.
  98  * - CD2X divider update. Single pipe can be active as the divider update
  99  *   can be synchronized with the pipe's start of vblank.
 100  * - Crawl the PLL smoothly to the new VCO frequency. Pipes can be active.
 101  * - Squash waveform update. Pipes can be active.
 102  * - Crawl and squash can also be done back to back. Pipes can be active.
 103  *
 104  * RAWCLK is a fixed frequency clock, often used by various auxiliary
 105  * blocks such as AUX CH or backlight PWM. Hence the only thing we
 106  * really need to know about RAWCLK is its frequency so that various
 107  * dividers can be programmed correctly.
 108  */
 109
 110 struct intel_cdclk_funcs {
 111         void (*get_cdclk)(struct drm_i915_private *i915,
 112                           struct intel_cdclk_config *cdclk_config);
 113         void (*set_cdclk)(struct drm_i915_private *i915,
 114                           const struct intel_cdclk_config *cdclk_config,
 115                           enum pipe pipe);
 116         int (*modeset_calc_cdclk)(struct intel_cdclk_state *state);
 117         u8 (*calc_voltage_level)(int cdclk);
 118 };
 119
 120 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
 121                            struct intel_cdclk_config *cdclk_config)
 122 {
 123         dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config);
 124 }
 125
 126 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
 127                                   const struct intel_cdclk_config *cdclk_config,
 128                                   enum pipe pipe)
 129 {
 130         dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe);
 131 }
 132
 133 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
 134                                           struct intel_cdclk_state *cdclk_config)
 135 {
 136         return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config);
 137 }
 138
 139 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
 140                                          int cdclk)
 141 {
 142         return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk);
 143 }
 144
 145 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
 146                                    struct intel_cdclk_config *cdclk_config)
 147 {
 148         cdclk_config->cdclk = 133333;
 149 }
 150
 151 static void fixed_200mhz_get_cdclk(struct drm_i915_private *dev_priv,
 152                                    struct intel_cdclk_config *cdclk_config)
 153 {
 154         cdclk_config->cdclk = 200000;
 155 }
 156
 157 static void fixed_266mhz_get_cdclk(struct drm_i915_private *dev_priv,
 158                                    struct intel_cdclk_config *cdclk_config)
 159 {
 160         cdclk_config->cdclk = 266667;
 161 }
 162
 163 static void fixed_333mhz_get_cdclk(struct drm_i915_private *dev_priv,
 164                                    struct intel_cdclk_config *cdclk_config)
 165 {
 166         cdclk_config->cdclk = 333333;
 167 }
 168
 169 static void fixed_400mhz_get_cdclk(struct drm_i915_private *dev_priv,
 170                                    struct intel_cdclk_config *cdclk_config)
 171 {
 172         cdclk_config->cdclk = 400000;
 173 }
 174
 175 static void fixed_450mhz_get_cdclk(struct drm_i915_private *dev_priv,
 176                                    struct intel_cdclk_config *cdclk_config)
 177 {
 178         cdclk_config->cdclk = 450000;
 179 }
 180
 181 static void i85x_get_cdclk(struct drm_i915_private *dev_priv,
 182                            struct intel_cdclk_config *cdclk_config)
 183 {
 184         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 185         u16 hpllcc = 0;
 186
 187         /*
 188          * 852GM/852GMV only supports 133 MHz and the HPLLCC
 189          * encoding is different :(
 190          * FIXME is this the right way to detect 852GM/852GMV?
 191          */
 192         if (pdev->revision == 0x1) {
 193                 cdclk_config->cdclk = 133333;
 194                 return;
 195         }
 196
 197         pci_bus_read_config_word(pdev->bus,
 198                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
 199
 200         /* Assume that the hardware is in the high speed state.  This
 201          * should be the default.
 202          */
 203         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
 204         case GC_CLOCK_133_200:
 205         case GC_CLOCK_133_200_2:
 206         case GC_CLOCK_100_200:
 207                 cdclk_config->cdclk = 200000;
 208                 break;
 209         case GC_CLOCK_166_250:
 210                 cdclk_config->cdclk = 250000;
 211                 break;
 212         case GC_CLOCK_100_133:
 213                 cdclk_config->cdclk = 133333;
 214                 break;
 215         case GC_CLOCK_133_266:
 216         case GC_CLOCK_133_266_2:
 217         case GC_CLOCK_166_266:
 218                 cdclk_config->cdclk = 266667;
 219                 break;
 220         }
 221 }
 222
 223 static void i915gm_get_cdclk(struct drm_i915_private *dev_priv,
 224                              struct intel_cdclk_config *cdclk_config)
 225 {
 226         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 227         u16 gcfgc = 0;
 228
 229         pci_read_config_word(pdev, GCFGC, &gcfgc);
 230
 231         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 232                 cdclk_config->cdclk = 133333;
 233                 return;
 234         }
 235
 236         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 237         case GC_DISPLAY_CLOCK_333_320_MHZ:
 238                 cdclk_config->cdclk = 333333;
 239                 break;
 240         default:
 241         case GC_DISPLAY_CLOCK_190_200_MHZ:
 242                 cdclk_config->cdclk = 190000;
 243                 break;
 244         }
 245 }
 246
 247 static void i945gm_get_cdclk(struct drm_i915_private *dev_priv,
 248                              struct intel_cdclk_config *cdclk_config)
 249 {
 250         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 251         u16 gcfgc = 0;
 252
 253         pci_read_config_word(pdev, GCFGC, &gcfgc);
 254
 255         if (gcfgc & GC_LOW_FREQUENCY_ENABLE) {
 256                 cdclk_config->cdclk = 133333;
 257                 return;
 258         }
 259
 260         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 261         case GC_DISPLAY_CLOCK_333_320_MHZ:
 262                 cdclk_config->cdclk = 320000;
 263                 break;
 264         default:
 265         case GC_DISPLAY_CLOCK_190_200_MHZ:
 266                 cdclk_config->cdclk = 200000;
 267                 break;
 268         }
 269 }
 270
 271 static unsigned int intel_hpll_vco(struct drm_i915_private *dev_priv)
 272 {
 273         static const unsigned int blb_vco[8] = {
 274                 [0] = 3200000,
 275                 [1] = 4000000,
 276                 [2] = 5333333,
 277                 [3] = 4800000,
 278                 [4] = 6400000,
 279         };
 280         static const unsigned int pnv_vco[8] = {
 281                 [0] = 3200000,
 282                 [1] = 4000000,
 283                 [2] = 5333333,
 284                 [3] = 4800000,
 285                 [4] = 2666667,
 286         };
 287         static const unsigned int cl_vco[8] = {
 288                 [0] = 3200000,
 289                 [1] = 4000000,
 290                 [2] = 5333333,
 291                 [3] = 6400000,
 292                 [4] = 3333333,
 293                 [5] = 3566667,
 294                 [6] = 4266667,
 295         };
 296         static const unsigned int elk_vco[8] = {
 297                 [0] = 3200000,
 298                 [1] = 4000000,
 299                 [2] = 5333333,
 300                 [3] = 4800000,
 301         };
 302         static const unsigned int ctg_vco[8] = {
 303                 [0] = 3200000,
 304                 [1] = 4000000,
 305                 [2] = 5333333,
 306                 [3] = 6400000,
 307                 [4] = 2666667,
 308                 [5] = 4266667,
 309         };
 310         const unsigned int *vco_table;
 311         unsigned int vco;
 312         u8 tmp = 0;
 313
 314         /* FIXME other chipsets? */
 315         if (IS_GM45(dev_priv))
 316                 vco_table = ctg_vco;
 317         else if (IS_G45(dev_priv))
 318                 vco_table = elk_vco;
 319         else if (IS_I965GM(dev_priv))
 320                 vco_table = cl_vco;
 321         else if (IS_PINEVIEW(dev_priv))
 322                 vco_table = pnv_vco;
 323         else if (IS_G33(dev_priv))
 324                 vco_table = blb_vco;
 325         else
 326                 return 0;
 327
 328         tmp = intel_de_read(dev_priv,
 329                             IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv) ? HPLLVCO_MOBILE : HPLLVCO);
 330
 331         vco = vco_table[tmp & 0x7];
 332         if (vco == 0)
 333                 drm_err(&dev_priv->drm, "Bad HPLL VCO (HPLLVCO=0x%02x)\n",
 334                         tmp);
 335         else
 336                 drm_dbg_kms(&dev_priv->drm, "HPLL VCO %u kHz\n", vco);
 337
 338         return vco;
 339 }
 340
 341 static void g33_get_cdclk(struct drm_i915_private *dev_priv,
 342                           struct intel_cdclk_config *cdclk_config)
 343 {
 344         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 345         static const u8 div_3200[] = { 12, 10,  8,  7, 5, 16 };
 346         static const u8 div_4000[] = { 14, 12, 10,  8, 6, 20 };
 347         static const u8 div_4800[] = { 20, 14, 12, 10, 8, 24 };
 348         static const u8 div_5333[] = { 20, 16, 12, 12, 8, 28 };
 349         const u8 *div_table;
 350         unsigned int cdclk_sel;
 351         u16 tmp = 0;
 352
 353         cdclk_config->vco = intel_hpll_vco(dev_priv);
 354
 355         pci_read_config_word(pdev, GCFGC, &tmp);
 356
 357         cdclk_sel = (tmp >> 4) & 0x7;
 358
 359         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 360                 goto fail;
 361
 362         switch (cdclk_config->vco) {
 363         case 3200000:
 364                 div_table = div_3200;
 365                 break;
 366         case 4000000:
 367                 div_table = div_4000;
 368                 break;
 369         case 4800000:
 370                 div_table = div_4800;
 371                 break;
 372         case 5333333:
 373                 div_table = div_5333;
 374                 break;
 375         default:
 376                 goto fail;
 377         }
 378
 379         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 380                                                 div_table[cdclk_sel]);
 381         return;
 382
 383 fail:
 384         drm_err(&dev_priv->drm,
 385                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n",
 386                 cdclk_config->vco, tmp);
 387         cdclk_config->cdclk = 190476;
 388 }
 389
 390 static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
 391                           struct intel_cdclk_config *cdclk_config)
 392 {
 393         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 394         u16 gcfgc = 0;
 395
 396         pci_read_config_word(pdev, GCFGC, &gcfgc);
 397
 398         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
 399         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
 400                 cdclk_config->cdclk = 266667;
 401                 break;
 402         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
 403                 cdclk_config->cdclk = 333333;
 404                 break;
 405         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
 406                 cdclk_config->cdclk = 444444;
 407                 break;
 408         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
 409                 cdclk_config->cdclk = 200000;
 410                 break;
 411         default:
 412                 drm_err(&dev_priv->drm,
 413                         "Unknown pnv display core clock 0x%04x\n", gcfgc);
 414                 fallthrough;
 415         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
 416                 cdclk_config->cdclk = 133333;
 417                 break;
 418         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
 419                 cdclk_config->cdclk = 166667;
 420                 break;
 421         }
 422 }
 423
 424 static void i965gm_get_cdclk(struct drm_i915_private *dev_priv,
 425                              struct intel_cdclk_config *cdclk_config)
 426 {
 427         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 428         static const u8 div_3200[] = { 16, 10,  8 };
 429         static const u8 div_4000[] = { 20, 12, 10 };
 430         static const u8 div_5333[] = { 24, 16, 14 };
 431         const u8 *div_table;
 432         unsigned int cdclk_sel;
 433         u16 tmp = 0;
 434
 435         cdclk_config->vco = intel_hpll_vco(dev_priv);
 436
 437         pci_read_config_word(pdev, GCFGC, &tmp);
 438
 439         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
 440
 441         if (cdclk_sel >= ARRAY_SIZE(div_3200))
 442                 goto fail;
 443
 444         switch (cdclk_config->vco) {
 445         case 3200000:
 446                 div_table = div_3200;
 447                 break;
 448         case 4000000:
 449                 div_table = div_4000;
 450                 break;
 451         case 5333333:
 452                 div_table = div_5333;
 453                 break;
 454         default:
 455                 goto fail;
 456         }
 457
 458         cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco,
 459                                                 div_table[cdclk_sel]);
 460         return;
 461
 462 fail:
 463         drm_err(&dev_priv->drm,
 464                 "Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n",
 465                 cdclk_config->vco, tmp);
 466         cdclk_config->cdclk = 200000;
 467 }
 468
 469 static void gm45_get_cdclk(struct drm_i915_private *dev_priv,
 470                            struct intel_cdclk_config *cdclk_config)
 471 {
 472         struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 473         unsigned int cdclk_sel;
 474         u16 tmp = 0;
 475
 476         cdclk_config->vco = intel_hpll_vco(dev_priv);
 477
 478         pci_read_config_word(pdev, GCFGC, &tmp);
 479
 480         cdclk_sel = (tmp >> 12) & 0x1;
 481
 482         switch (cdclk_config->vco) {
 483         case 2666667:
 484         case 4000000:
 485         case 5333333:
 486                 cdclk_config->cdclk = cdclk_sel ? 333333 : 222222;
 487                 break;
 488         case 3200000:
 489                 cdclk_config->cdclk = cdclk_sel ? 320000 : 228571;
 490                 break;
 491         default:
 492                 drm_err(&dev_priv->drm,
 493                         "Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n",
 494                         cdclk_config->vco, tmp);
 495                 cdclk_config->cdclk = 222222;
 496                 break;
 497         }
 498 }
 499
 500 static void hsw_get_cdclk(struct drm_i915_private *dev_priv,
 501                           struct intel_cdclk_config *cdclk_config)
 502 {
 503         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 504         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 505
 506         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 507                 cdclk_config->cdclk = 800000;
 508         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 509                 cdclk_config->cdclk = 450000;
 510         else if (freq == LCPLL_CLK_FREQ_450)
 511                 cdclk_config->cdclk = 450000;
 512         else if (IS_HASWELL_ULT(dev_priv))
 513                 cdclk_config->cdclk = 337500;
 514         else
 515                 cdclk_config->cdclk = 540000;
 516 }
 517
 518 static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 519 {
 520         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ?
 521                 333333 : 320000;
 522
 523         /*
 524          * We seem to get an unstable or solid color picture at 200MHz.
 525          * Not sure what's wrong. For now use 200MHz only when all pipes
 526          * are off.
 527          */
 528         if (IS_VALLEYVIEW(dev_priv) && min_cdclk > freq_320)
 529                 return 400000;
 530         else if (min_cdclk > 266667)
 531                 return freq_320;
 532         else if (min_cdclk > 0)
 533                 return 266667;
 534         else
 535                 return 200000;
 536 }
 537
 538 static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
 539 {
 540         if (IS_VALLEYVIEW(dev_priv)) {
 541                 if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
 542                         return 2;
 543                 else if (cdclk >= 266667)
 544                         return 1;
 545                 else
 546                         return 0;
 547         } else {
 548                 /*
 549                  * Specs are full of misinformation, but testing on actual
 550                  * hardware has shown that we just need to write the desired
 551                  * CCK divider into the Punit register.
 552                  */
 553                 return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
 554         }
 555 }
 556
 557 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
 558                           struct intel_cdclk_config *cdclk_config)
 559 {
 560         u32 val;
 561
 562         vlv_iosf_sb_get(dev_priv,
 563                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 564
 565         cdclk_config->vco = vlv_get_hpll_vco(dev_priv);
 566         cdclk_config->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
 567                                                 CCK_DISPLAY_CLOCK_CONTROL,
 568                                                 cdclk_config->vco);
 569
 570         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 571
 572         vlv_iosf_sb_put(dev_priv,
 573                         BIT(VLV_IOSF_SB_CCK) | BIT(VLV_IOSF_SB_PUNIT));
 574
 575         if (IS_VALLEYVIEW(dev_priv))
 576                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
 577                         DSPFREQGUAR_SHIFT;
 578         else
 579                 cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
 580                         DSPFREQGUAR_SHIFT_CHV;
 581 }
 582
 583 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 584 {
 585         unsigned int credits, default_credits;
 586
 587         if (IS_CHERRYVIEW(dev_priv))
 588                 default_credits = PFI_CREDIT(12);
 589         else
 590                 default_credits = PFI_CREDIT(8);
 591
 592         if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 593                 /* CHV suggested value is 31 or 63 */
 594                 if (IS_CHERRYVIEW(dev_priv))
 595                         credits = PFI_CREDIT_63;
 596                 else
 597                         credits = PFI_CREDIT(15);
 598         } else {
 599                 credits = default_credits;
 600         }
 601
 602         /*
 603          * WA - write default credits before re-programming
 604          * FIXME: should we also set the resend bit here?
 605          */
 606         intel_de_write(dev_priv, GCI_CONTROL,
 607                        VGA_FAST_MODE_DISABLE | default_credits);
 608
 609         intel_de_write(dev_priv, GCI_CONTROL,
 610                        VGA_FAST_MODE_DISABLE | credits | PFI_CREDIT_RESEND);
 611
 612         /*
 613          * FIXME is this guaranteed to clear
 614          * immediately or should we poll for it?
 615          */
 616         drm_WARN_ON(&dev_priv->drm,
 617                     intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND);
 618 }
 619
 620 static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
 621                           const struct intel_cdclk_config *cdclk_config,
 622                           enum pipe pipe)
 623 {
 624         int cdclk = cdclk_config->cdclk;
 625         u32 val, cmd = cdclk_config->voltage_level;
 626         intel_wakeref_t wakeref;
 627
 628         switch (cdclk) {
 629         case 400000:
 630         case 333333:
 631         case 320000:
 632         case 266667:
 633         case 200000:
 634                 break;
 635         default:
 636                 MISSING_CASE(cdclk);
 637                 return;
 638         }
 639
 640         /* There are cases where we can end up here with power domains
 641          * off and a CDCLK frequency other than the minimum, like when
 642          * issuing a modeset without actually changing any display after
 643          * a system suspend.  So grab the display core domain, which covers
 644          * the HW blocks needed for the following programming.
 645          */
 646         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 647
 648         vlv_iosf_sb_get(dev_priv,
 649                         BIT(VLV_IOSF_SB_CCK) |
 650                         BIT(VLV_IOSF_SB_BUNIT) |
 651                         BIT(VLV_IOSF_SB_PUNIT));
 652
 653         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 654         val &= ~DSPFREQGUAR_MASK;
 655         val |= (cmd << DSPFREQGUAR_SHIFT);
 656         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 657         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 658                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 659                      50)) {
 660                 drm_err(&dev_priv->drm,
 661                         "timed out waiting for CDclk change\n");
 662         }
 663
 664         if (cdclk == 400000) {
 665                 u32 divider;
 666
 667                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1,
 668                                             cdclk) - 1;
 669
 670                 /* adjust cdclk divider */
 671                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
 672                 val &= ~CCK_FREQUENCY_VALUES;
 673                 val |= divider;
 674                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
 675
 676                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
 677                               CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 678                              50))
 679                         drm_err(&dev_priv->drm,
 680                                 "timed out waiting for CDclk change\n");
 681         }
 682
 683         /* adjust self-refresh exit latency value */
 684         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
 685         val &= ~0x7f;
 686
 687         /*
 688          * For high bandwidth configs, we set a higher latency in the bunit
 689          * so that the core display fetch happens in time to avoid underruns.
 690          */
 691         if (cdclk == 400000)
 692                 val |= 4500 / 250; /* 4.5 usec */
 693         else
 694                 val |= 3000 / 250; /* 3.0 usec */
 695         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
 696
 697         vlv_iosf_sb_put(dev_priv,
 698                         BIT(VLV_IOSF_SB_CCK) |
 699                         BIT(VLV_IOSF_SB_BUNIT) |
 700                         BIT(VLV_IOSF_SB_PUNIT));
 701
 702         intel_update_cdclk(dev_priv);
 703
 704         vlv_program_pfi_credits(dev_priv);
 705
 706         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 707 }
 708
 709 static void chv_set_cdclk(struct drm_i915_private *dev_priv,
 710                           const struct intel_cdclk_config *cdclk_config,
 711                           enum pipe pipe)
 712 {
 713         int cdclk = cdclk_config->cdclk;
 714         u32 val, cmd = cdclk_config->voltage_level;
 715         intel_wakeref_t wakeref;
 716
 717         switch (cdclk) {
 718         case 333333:
 719         case 320000:
 720         case 266667:
 721         case 200000:
 722                 break;
 723         default:
 724                 MISSING_CASE(cdclk);
 725                 return;
 726         }
 727
 728         /* There are cases where we can end up here with power domains
 729          * off and a CDCLK frequency other than the minimum, like when
 730          * issuing a modeset without actually changing any display after
 731          * a system suspend.  So grab the display core domain, which covers
 732          * the HW blocks needed for the following programming.
 733          */
 734         wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
 735
 736         vlv_punit_get(dev_priv);
 737         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 738         val &= ~DSPFREQGUAR_MASK_CHV;
 739         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
 740         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 741         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) &
 742                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 743                      50)) {
 744                 drm_err(&dev_priv->drm,
 745                         "timed out waiting for CDclk change\n");
 746         }
 747
 748         vlv_punit_put(dev_priv);
 749
 750         intel_update_cdclk(dev_priv);
 751
 752         vlv_program_pfi_credits(dev_priv);
 753
 754         intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 755 }
 756
 757 static int bdw_calc_cdclk(int min_cdclk)
 758 {
 759         if (min_cdclk > 540000)
 760                 return 675000;
 761         else if (min_cdclk > 450000)
 762                 return 540000;
 763         else if (min_cdclk > 337500)
 764                 return 450000;
 765         else
 766                 return 337500;
 767 }
 768
 769 static u8 bdw_calc_voltage_level(int cdclk)
 770 {
 771         switch (cdclk) {
 772         default:
 773         case 337500:
 774                 return 2;
 775         case 450000:
 776                 return 0;
 777         case 540000:
 778                 return 1;
 779         case 675000:
 780                 return 3;
 781         }
 782 }
 783
 784 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
 785                           struct intel_cdclk_config *cdclk_config)
 786 {
 787         u32 lcpll = intel_de_read(dev_priv, LCPLL_CTL);
 788         u32 freq = lcpll & LCPLL_CLK_FREQ_MASK;
 789
 790         if (lcpll & LCPLL_CD_SOURCE_FCLK)
 791                 cdclk_config->cdclk = 800000;
 792         else if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
 793                 cdclk_config->cdclk = 450000;
 794         else if (freq == LCPLL_CLK_FREQ_450)
 795                 cdclk_config->cdclk = 450000;
 796         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
 797                 cdclk_config->cdclk = 540000;
 798         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
 799                 cdclk_config->cdclk = 337500;
 800         else
 801                 cdclk_config->cdclk = 675000;
 802
 803         /*
 804          * Can't read this out :( Let's assume it's
 805          * at least what the CDCLK frequency requires.
 806          */
 807         cdclk_config->voltage_level =
 808                 bdw_calc_voltage_level(cdclk_config->cdclk);
 809 }
 810
 811 static u32 bdw_cdclk_freq_sel(int cdclk)
 812 {
 813         switch (cdclk) {
 814         default:
 815                 MISSING_CASE(cdclk);
 816                 fallthrough;
 817         case 337500:
 818                 return LCPLL_CLK_FREQ_337_5_BDW;
 819         case 450000:
 820                 return LCPLL_CLK_FREQ_450;
 821         case 540000:
 822                 return LCPLL_CLK_FREQ_54O_BDW;
 823         case 675000:
 824                 return LCPLL_CLK_FREQ_675_BDW;
 825         }
 826 }
 827
 828 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
 829                           const struct intel_cdclk_config *cdclk_config,
 830                           enum pipe pipe)
 831 {
 832         int cdclk = cdclk_config->cdclk;
 833         int ret;
 834
 835         if (drm_WARN(&dev_priv->drm,
 836                      (intel_de_read(dev_priv, LCPLL_CTL) &
 837                       (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
 838                        LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
 839                        LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
 840                        LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
 841                      "trying to change cdclk frequency with cdclk not enabled\n"))
 842                 return;
 843
 844         ret = snb_pcode_write(&dev_priv->uncore, BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
 845         if (ret) {
 846                 drm_err(&dev_priv->drm,
 847                         "failed to inform pcode about cdclk change\n");
 848                 return;
 849         }
 850
 851         intel_de_rmw(dev_priv, LCPLL_CTL,
 852                      0, LCPLL_CD_SOURCE_FCLK);
 853
 854         /*
 855          * According to the spec, it should be enough to poll for this 1 us.
 856          * However, extensive testing shows that this can take longer.
 857          */
 858         if (wait_for_us(intel_de_read(dev_priv, LCPLL_CTL) &
 859                         LCPLL_CD_SOURCE_FCLK_DONE, 100))
 860                 drm_err(&dev_priv->drm, "Switching to FCLK failed\n");
 861
 862         intel_de_rmw(dev_priv, LCPLL_CTL,
 863                      LCPLL_CLK_FREQ_MASK, bdw_cdclk_freq_sel(cdclk));
 864
 865         intel_de_rmw(dev_priv, LCPLL_CTL,
 866                      LCPLL_CD_SOURCE_FCLK, 0);
 867
 868         if (wait_for_us((intel_de_read(dev_priv, LCPLL_CTL) &
 869                          LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
 870                 drm_err(&dev_priv->drm, "Switching back to LCPLL failed\n");
 871
 872         snb_pcode_write(&dev_priv->uncore, HSW_PCODE_DE_WRITE_FREQ_REQ,
 873                         cdclk_config->voltage_level);
 874
 875         intel_de_write(dev_priv, CDCLK_FREQ,
 876                        DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 877
 878         intel_update_cdclk(dev_priv);
 879 }
 880
 881 static int skl_calc_cdclk(int min_cdclk, int vco)
 882 {
 883         if (vco == 8640000) {
 884                 if (min_cdclk > 540000)
 885                         return 617143;
 886                 else if (min_cdclk > 432000)
 887                         return 540000;
 888                 else if (min_cdclk > 308571)
 889                         return 432000;
 890                 else
 891                         return 308571;
 892         } else {
 893                 if (min_cdclk > 540000)
 894                         return 675000;
 895                 else if (min_cdclk > 450000)
 896                         return 540000;
 897                 else if (min_cdclk > 337500)
 898                         return 450000;
 899                 else
 900                         return 337500;
 901         }
 902 }
 903
 904 static u8 skl_calc_voltage_level(int cdclk)
 905 {
 906         if (cdclk > 540000)
 907                 return 3;
 908         else if (cdclk > 450000)
 909                 return 2;
 910         else if (cdclk > 337500)
 911                 return 1;
 912         else
 913                 return 0;
 914 }
 915
 916 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
 917                              struct intel_cdclk_config *cdclk_config)
 918 {
 919         u32 val;
 920
 921         cdclk_config->ref = 24000;
 922         cdclk_config->vco = 0;
 923
 924         val = intel_de_read(dev_priv, LCPLL1_CTL);
 925         if ((val & LCPLL_PLL_ENABLE) == 0)
 926                 return;
 927
 928         if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0))
 929                 return;
 930
 931         val = intel_de_read(dev_priv, DPLL_CTRL1);
 932
 933         if (drm_WARN_ON(&dev_priv->drm,
 934                         (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
 935                                 DPLL_CTRL1_SSC(SKL_DPLL0) |
 936                                 DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
 937                         DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
 938                 return;
 939
 940         switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
 941         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
 942         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
 943         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
 944         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
 945                 cdclk_config->vco = 8100000;
 946                 break;
 947         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
 948         case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
 949                 cdclk_config->vco = 8640000;
 950                 break;
 951         default:
 952                 MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
 953                 break;
 954         }
 955 }
 956
 957 static void skl_get_cdclk(struct drm_i915_private *dev_priv,
 958                           struct intel_cdclk_config *cdclk_config)
 959 {
 960         u32 cdctl;
 961
 962         skl_dpll0_update(dev_priv, cdclk_config);
 963
 964         cdclk_config->cdclk = cdclk_config->bypass = cdclk_config->ref;
 965
 966         if (cdclk_config->vco == 0)
 967                 goto out;
 968
 969         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 970
 971         if (cdclk_config->vco == 8640000) {
 972                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 973                 case CDCLK_FREQ_450_432:
 974                         cdclk_config->cdclk = 432000;
 975                         break;
 976                 case CDCLK_FREQ_337_308:
 977                         cdclk_config->cdclk = 308571;
 978                         break;
 979                 case CDCLK_FREQ_540:
 980                         cdclk_config->cdclk = 540000;
 981                         break;
 982                 case CDCLK_FREQ_675_617:
 983                         cdclk_config->cdclk = 617143;
 984                         break;
 985                 default:
 986                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
 987                         break;
 988                 }
 989         } else {
 990                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
 991                 case CDCLK_FREQ_450_432:
 992                         cdclk_config->cdclk = 450000;
 993                         break;
 994                 case CDCLK_FREQ_337_308:
 995                         cdclk_config->cdclk = 337500;
 996                         break;
 997                 case CDCLK_FREQ_540:
 998                         cdclk_config->cdclk = 540000;
 999                         break;
1000                 case CDCLK_FREQ_675_617:
1001                         cdclk_config->cdclk = 675000;
1002                         break;
1003                 default:
1004                         MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
1005                         break;
1006                 }
1007         }
1008
1009  out:
1010         /*
1011          * Can't read this out :( Let's assume it's
1012          * at least what the CDCLK frequency requires.
1013          */
1014         cdclk_config->voltage_level =
1015                 skl_calc_voltage_level(cdclk_config->cdclk);
1016 }
1017
1018 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
1019 static int skl_cdclk_decimal(int cdclk)
1020 {
1021         return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
1022 }
1023
1024 static void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv,
1025                                         int vco)
1026 {
1027         bool changed = dev_priv->skl_preferred_vco_freq != vco;
1028
1029         dev_priv->skl_preferred_vco_freq = vco;
1030
1031         if (changed)
1032                 intel_update_max_cdclk(dev_priv);
1033 }
1034
1035 static u32 skl_dpll0_link_rate(struct drm_i915_private *dev_priv, int vco)
1036 {
1037         drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
1038
1039         /*
1040          * We always enable DPLL0 with the lowest link rate possible, but still
1041          * taking into account the VCO required to operate the eDP panel at the
1042          * desired frequency. The usual DP link rates operate with a VCO of
1043          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
1044          * The modeset code is responsible for the selection of the exact link
1045          * rate later on, with the constraint of choosing a frequency that
1046          * works with vco.
1047          */
1048         if (vco == 8640000)
1049                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0);
1050         else
1051                 return DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0);
1052 }
1053
1054 static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
1055 {
1056         intel_de_rmw(dev_priv, DPLL_CTRL1,
1057                      DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
1058                      DPLL_CTRL1_SSC(SKL_DPLL0) |
1059                      DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0),
1060                      DPLL_CTRL1_OVERRIDE(SKL_DPLL0) |
1061                      skl_dpll0_link_rate(dev_priv, vco));
1062         intel_de_posting_read(dev_priv, DPLL_CTRL1);
1063
1064         intel_de_rmw(dev_priv, LCPLL1_CTL,
1065                      0, LCPLL_PLL_ENABLE);
1066
1067         if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
1068                 drm_err(&dev_priv->drm, "DPLL0 not locked\n");
1069
1070         dev_priv->display.cdclk.hw.vco = vco;
1071
1072         /* We'll want to keep using the current vco from now on. */
1073         skl_set_preferred_cdclk_vco(dev_priv, vco);
1074 }
1075
1076 static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
1077 {
1078         intel_de_rmw(dev_priv, LCPLL1_CTL,
1079                      LCPLL_PLL_ENABLE, 0);
1080
1081         if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
1082                 drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
1083
1084         dev_priv->display.cdclk.hw.vco = 0;
1085 }
1086
1087 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
1088                               int cdclk, int vco)
1089 {
1090         switch (cdclk) {
1091         default:
1092                 drm_WARN_ON(&dev_priv->drm,
1093                             cdclk != dev_priv->display.cdclk.hw.bypass);
1094                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1095                 fallthrough;
1096         case 308571:
1097         case 337500:
1098                 return CDCLK_FREQ_337_308;
1099         case 450000:
1100         case 432000:
1101                 return CDCLK_FREQ_450_432;
1102         case 540000:
1103                 return CDCLK_FREQ_540;
1104         case 617143:
1105         case 675000:
1106                 return CDCLK_FREQ_675_617;
1107         }
1108 }
1109
1110 static void skl_set_cdclk(struct drm_i915_private *dev_priv,
1111                           const struct intel_cdclk_config *cdclk_config,
1112                           enum pipe pipe)
1113 {
1114         int cdclk = cdclk_config->cdclk;
1115         int vco = cdclk_config->vco;
1116         u32 freq_select, cdclk_ctl;
1117         int ret;
1118
1119         /*
1120          * Based on WA#1183 CDCLK rates 308 and 617MHz CDCLK rates are
1121          * unsupported on SKL. In theory this should never happen since only
1122          * the eDP1.4 2.16 and 4.32Gbps rates require it, but eDP1.4 is not
1123          * supported on SKL either, see the above WA. WARN whenever trying to
1124          * use the corresponding VCO freq as that always leads to using the
1125          * minimum 308MHz CDCLK.
1126          */
1127         drm_WARN_ON_ONCE(&dev_priv->drm,
1128                          IS_SKYLAKE(dev_priv) && vco == 8640000);
1129
1130         ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1131                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
1132                                 SKL_CDCLK_READY_FOR_CHANGE,
1133                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
1134         if (ret) {
1135                 drm_err(&dev_priv->drm,
1136                         "Failed to inform PCU about cdclk change (%d)\n", ret);
1137                 return;
1138         }
1139
1140         freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
1141
1142         if (dev_priv->display.cdclk.hw.vco != 0 &&
1143             dev_priv->display.cdclk.hw.vco != vco)
1144                 skl_dpll0_disable(dev_priv);
1145
1146         cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
1147
1148         if (dev_priv->display.cdclk.hw.vco != vco) {
1149                 /* Wa Display #1183: skl,kbl,cfl */
1150                 cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1151                 cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1152                 intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1153         }
1154
1155         /* Wa Display #1183: skl,kbl,cfl */
1156         cdclk_ctl |= CDCLK_DIVMUX_CD_OVERRIDE;
1157         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1158         intel_de_posting_read(dev_priv, CDCLK_CTL);
1159
1160         if (dev_priv->display.cdclk.hw.vco != vco)
1161                 skl_dpll0_enable(dev_priv, vco);
1162
1163         /* Wa Display #1183: skl,kbl,cfl */
1164         cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
1165         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1166
1167         cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
1168         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1169
1170         /* Wa Display #1183: skl,kbl,cfl */
1171         cdclk_ctl &= ~CDCLK_DIVMUX_CD_OVERRIDE;
1172         intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
1173         intel_de_posting_read(dev_priv, CDCLK_CTL);
1174
1175         /* inform PCU of the change */
1176         snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
1177                         cdclk_config->voltage_level);
1178
1179         intel_update_cdclk(dev_priv);
1180 }
1181
1182 static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
1183 {
1184         u32 cdctl, expected;
1185
1186         /*
1187          * check if the pre-os initialized the display
1188          * There is SWF18 scratchpad register defined which is set by the
1189          * pre-os which can be used by the OS drivers to check the status
1190          */
1191         if ((intel_de_read(dev_priv, SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
1192                 goto sanitize;
1193
1194         intel_update_cdclk(dev_priv);
1195         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
1196
1197         /* Is PLL enabled and locked ? */
1198         if (dev_priv->display.cdclk.hw.vco == 0 ||
1199             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
1200                 goto sanitize;
1201
1202         /* DPLL okay; verify the cdclock
1203          *
1204          * Noticed in some instances that the freq selection is correct but
1205          * decimal part is programmed wrong from BIOS where pre-os does not
1206          * enable display. Verify the same as well.
1207          */
1208         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
1209         expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
1210                 skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk);
1211         if (cdctl == expected)
1212                 /* All well; nothing to sanitize */
1213                 return;
1214
1215 sanitize:
1216         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
1217
1218         /* force cdclk programming */
1219         dev_priv->display.cdclk.hw.cdclk = 0;
1220         /* force full PLL disable + enable */
1221         dev_priv->display.cdclk.hw.vco = ~0;
1222 }
1223
1224 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
1225 {
1226         struct intel_cdclk_config cdclk_config;
1227
1228         skl_sanitize_cdclk(dev_priv);
1229
1230         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
1231             dev_priv->display.cdclk.hw.vco != 0) {
1232                 /*
1233                  * Use the current vco as our initial
1234                  * guess as to what the preferred vco is.
1235                  */
1236                 if (dev_priv->skl_preferred_vco_freq == 0)
1237                         skl_set_preferred_cdclk_vco(dev_priv,
1238                                                     dev_priv->display.cdclk.hw.vco);
1239                 return;
1240         }
1241
1242         cdclk_config = dev_priv->display.cdclk.hw;
1243
1244         cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
1245         if (cdclk_config.vco == 0)
1246                 cdclk_config.vco = 8100000;
1247         cdclk_config.cdclk = skl_calc_cdclk(0, cdclk_config.vco);
1248         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1249
1250         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1251 }
1252
1253 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
1254 {
1255         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
1256
1257         cdclk_config.cdclk = cdclk_config.bypass;
1258         cdclk_config.vco = 0;
1259         cdclk_config.voltage_level = skl_calc_voltage_level(cdclk_config.cdclk);
1260
1261         skl_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
1262 }
1263
1264 struct intel_cdclk_vals {
1265         u32 cdclk;
1266         u16 refclk;
1267         u16 waveform;
1268         u8 ratio;
1269 };
1270
1271 static const struct intel_cdclk_vals bxt_cdclk_table[] = {
1272         { .refclk = 19200, .cdclk = 144000, .ratio = 60 },
1273         { .refclk = 19200, .cdclk = 288000, .ratio = 60 },
1274         { .refclk = 19200, .cdclk = 384000, .ratio = 60 },
1275         { .refclk = 19200, .cdclk = 576000, .ratio = 60 },
1276         { .refclk = 19200, .cdclk = 624000, .ratio = 65 },
1277         {}
1278 };
1279
1280 static const struct intel_cdclk_vals glk_cdclk_table[] = {
1281         { .refclk = 19200, .cdclk =  79200, .ratio = 33 },
1282         { .refclk = 19200, .cdclk = 158400, .ratio = 33 },
1283         { .refclk = 19200, .cdclk = 316800, .ratio = 33 },
1284         {}
1285 };
1286
1287 static const struct intel_cdclk_vals icl_cdclk_table[] = {
1288         { .refclk = 19200, .cdclk = 172800, .ratio = 18 },
1289         { .refclk = 19200, .cdclk = 192000, .ratio = 20 },
1290         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1291         { .refclk = 19200, .cdclk = 326400, .ratio = 68 },
1292         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1293         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1294
1295         { .refclk = 24000, .cdclk = 180000, .ratio = 15 },
1296         { .refclk = 24000, .cdclk = 192000, .ratio = 16 },
1297         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1298         { .refclk = 24000, .cdclk = 324000, .ratio = 54 },
1299         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1300         { .refclk = 24000, .cdclk = 648000, .ratio = 54 },
1301
1302         { .refclk = 38400, .cdclk = 172800, .ratio =  9 },
1303         { .refclk = 38400, .cdclk = 192000, .ratio = 10 },
1304         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1305         { .refclk = 38400, .cdclk = 326400, .ratio = 34 },
1306         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1307         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1308         {}
1309 };
1310
1311 static const struct intel_cdclk_vals rkl_cdclk_table[] = {
1312         { .refclk = 19200, .cdclk = 172800, .ratio =  36 },
1313         { .refclk = 19200, .cdclk = 192000, .ratio =  40 },
1314         { .refclk = 19200, .cdclk = 307200, .ratio =  64 },
1315         { .refclk = 19200, .cdclk = 326400, .ratio = 136 },
1316         { .refclk = 19200, .cdclk = 556800, .ratio = 116 },
1317         { .refclk = 19200, .cdclk = 652800, .ratio = 136 },
1318
1319         { .refclk = 24000, .cdclk = 180000, .ratio =  30 },
1320         { .refclk = 24000, .cdclk = 192000, .ratio =  32 },
1321         { .refclk = 24000, .cdclk = 312000, .ratio =  52 },
1322         { .refclk = 24000, .cdclk = 324000, .ratio = 108 },
1323         { .refclk = 24000, .cdclk = 552000, .ratio =  92 },
1324         { .refclk = 24000, .cdclk = 648000, .ratio = 108 },
1325
1326         { .refclk = 38400, .cdclk = 172800, .ratio = 18 },
1327         { .refclk = 38400, .cdclk = 192000, .ratio = 20 },
1328         { .refclk = 38400, .cdclk = 307200, .ratio = 32 },
1329         { .refclk = 38400, .cdclk = 326400, .ratio = 68 },
1330         { .refclk = 38400, .cdclk = 556800, .ratio = 58 },
1331         { .refclk = 38400, .cdclk = 652800, .ratio = 68 },
1332         {}
1333 };
1334
1335 static const struct intel_cdclk_vals adlp_a_step_cdclk_table[] = {
1336         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1337         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1338         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1339
1340         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1341         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1342         { .refclk = 24400, .cdclk = 648000, .ratio = 54 },
1343
1344         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1345         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1346         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1347         {}
1348 };
1349
1350 static const struct intel_cdclk_vals adlp_cdclk_table[] = {
1351         { .refclk = 19200, .cdclk = 172800, .ratio = 27 },
1352         { .refclk = 19200, .cdclk = 192000, .ratio = 20 },
1353         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1354         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1355         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1356
1357         { .refclk = 24000, .cdclk = 176000, .ratio = 22 },
1358         { .refclk = 24000, .cdclk = 192000, .ratio = 16 },
1359         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1360         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1361         { .refclk = 24000, .cdclk = 648000, .ratio = 54 },
1362
1363         { .refclk = 38400, .cdclk = 179200, .ratio = 14 },
1364         { .refclk = 38400, .cdclk = 192000, .ratio = 10 },
1365         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1366         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1367         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1368         {}
1369 };
1370
1371 static const struct intel_cdclk_vals rplu_cdclk_table[] = {
1372         { .refclk = 19200, .cdclk = 172800, .ratio = 27 },
1373         { .refclk = 19200, .cdclk = 192000, .ratio = 20 },
1374         { .refclk = 19200, .cdclk = 307200, .ratio = 32 },
1375         { .refclk = 19200, .cdclk = 480000, .ratio = 50 },
1376         { .refclk = 19200, .cdclk = 556800, .ratio = 58 },
1377         { .refclk = 19200, .cdclk = 652800, .ratio = 68 },
1378
1379         { .refclk = 24000, .cdclk = 176000, .ratio = 22 },
1380         { .refclk = 24000, .cdclk = 192000, .ratio = 16 },
1381         { .refclk = 24000, .cdclk = 312000, .ratio = 26 },
1382         { .refclk = 24000, .cdclk = 480000, .ratio = 40 },
1383         { .refclk = 24000, .cdclk = 552000, .ratio = 46 },
1384         { .refclk = 24000, .cdclk = 648000, .ratio = 54 },
1385
1386         { .refclk = 38400, .cdclk = 179200, .ratio = 14 },
1387         { .refclk = 38400, .cdclk = 192000, .ratio = 10 },
1388         { .refclk = 38400, .cdclk = 307200, .ratio = 16 },
1389         { .refclk = 38400, .cdclk = 480000, .ratio = 25 },
1390         { .refclk = 38400, .cdclk = 556800, .ratio = 29 },
1391         { .refclk = 38400, .cdclk = 652800, .ratio = 34 },
1392         {}
1393 };
1394
1395 static const struct intel_cdclk_vals dg2_cdclk_table[] = {
1396         { .refclk = 38400, .cdclk = 163200, .ratio = 34, .waveform = 0x8888 },
1397         { .refclk = 38400, .cdclk = 204000, .ratio = 34, .waveform = 0x9248 },
1398         { .refclk = 38400, .cdclk = 244800, .ratio = 34, .waveform = 0xa4a4 },
1399         { .refclk = 38400, .cdclk = 285600, .ratio = 34, .waveform = 0xa54a },
1400         { .refclk = 38400, .cdclk = 326400, .ratio = 34, .waveform = 0xaaaa },
1401         { .refclk = 38400, .cdclk = 367200, .ratio = 34, .waveform = 0xad5a },
1402         { .refclk = 38400, .cdclk = 408000, .ratio = 34, .waveform = 0xb6b6 },
1403         { .refclk = 38400, .cdclk = 448800, .ratio = 34, .waveform = 0xdbb6 },
1404         { .refclk = 38400, .cdclk = 489600, .ratio = 34, .waveform = 0xeeee },
1405         { .refclk = 38400, .cdclk = 530400, .ratio = 34, .waveform = 0xf7de },
1406         { .refclk = 38400, .cdclk = 571200, .ratio = 34, .waveform = 0xfefe },
1407         { .refclk = 38400, .cdclk = 612000, .ratio = 34, .waveform = 0xfffe },
1408         { .refclk = 38400, .cdclk = 652800, .ratio = 34, .waveform = 0xffff },
1409         {}
1410 };
1411
1412 static const struct intel_cdclk_vals mtl_cdclk_table[] = {
1413         { .refclk = 38400, .cdclk = 172800, .ratio = 16, .waveform = 0xad5a },
1414         { .refclk = 38400, .cdclk = 192000, .ratio = 16, .waveform = 0xb6b6 },
1415         { .refclk = 38400, .cdclk = 307200, .ratio = 16, .waveform = 0x0000 },
1416         { .refclk = 38400, .cdclk = 480000, .ratio = 25, .waveform = 0x0000 },
1417         { .refclk = 38400, .cdclk = 556800, .ratio = 29, .waveform = 0x0000 },
1418         { .refclk = 38400, .cdclk = 652800, .ratio = 34, .waveform = 0x0000 },
1419         {}
1420 };
1421
1422 static const struct intel_cdclk_vals xe2lpd_cdclk_table[] = {
1423         { .refclk = 38400, .cdclk = 153600, .ratio = 16, .waveform = 0xaaaa },
1424         { .refclk = 38400, .cdclk = 172800, .ratio = 16, .waveform = 0xad5a },
1425         { .refclk = 38400, .cdclk = 192000, .ratio = 16, .waveform = 0xb6b6 },
1426         { .refclk = 38400, .cdclk = 211200, .ratio = 16, .waveform = 0xdbb6 },
1427         { .refclk = 38400, .cdclk = 230400, .ratio = 16, .waveform = 0xeeee },
1428         { .refclk = 38400, .cdclk = 249600, .ratio = 16, .waveform = 0xf7de },
1429         { .refclk = 38400, .cdclk = 268800, .ratio = 16, .waveform = 0xfefe },
1430         { .refclk = 38400, .cdclk = 288000, .ratio = 16, .waveform = 0xfffe },
1431         { .refclk = 38400, .cdclk = 307200, .ratio = 16, .waveform = 0xffff },
1432         { .refclk = 38400, .cdclk = 330000, .ratio = 25, .waveform = 0xdbb6 },
1433         { .refclk = 38400, .cdclk = 360000, .ratio = 25, .waveform = 0xeeee },
1434         { .refclk = 38400, .cdclk = 390000, .ratio = 25, .waveform = 0xf7de },
1435         { .refclk = 38400, .cdclk = 420000, .ratio = 25, .waveform = 0xfefe },
1436         { .refclk = 38400, .cdclk = 450000, .ratio = 25, .waveform = 0xfffe },
1437         { .refclk = 38400, .cdclk = 480000, .ratio = 25, .waveform = 0xffff },
1438         { .refclk = 38400, .cdclk = 487200, .ratio = 29, .waveform = 0xfefe },
1439         { .refclk = 38400, .cdclk = 522000, .ratio = 29, .waveform = 0xfffe },
1440         { .refclk = 38400, .cdclk = 556800, .ratio = 29, .waveform = 0xffff },
1441         { .refclk = 38400, .cdclk = 571200, .ratio = 34, .waveform = 0xfefe },
1442         { .refclk = 38400, .cdclk = 612000, .ratio = 34, .waveform = 0xfffe },
1443         { .refclk = 38400, .cdclk = 652800, .ratio = 34, .waveform = 0xffff },
1444         {}
1445 };
1446
1447 static const int cdclk_squash_len = 16;
1448
1449 static int cdclk_squash_divider(u16 waveform)
1450 {
1451         return hweight16(waveform ?: 0xffff);
1452 }
1453
1454 static int cdclk_divider(int cdclk, int vco, u16 waveform)
1455 {
1456         /* 2 * cd2x divider */
1457         return DIV_ROUND_CLOSEST(vco * cdclk_squash_divider(waveform),
1458                                  cdclk * cdclk_squash_len);
1459 }
1460
1461 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
1462 {
1463         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1464         int i;
1465
1466         for (i = 0; table[i].refclk; i++)
1467                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1468                     table[i].cdclk >= min_cdclk)
1469                         return table[i].cdclk;
1470
1471         drm_WARN(&dev_priv->drm, 1,
1472                  "Cannot satisfy minimum cdclk %d with refclk %u\n",
1473                  min_cdclk, dev_priv->display.cdclk.hw.ref);
1474         return 0;
1475 }
1476
1477 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
1478 {
1479         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1480         int i;
1481
1482         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1483                 return 0;
1484
1485         for (i = 0; table[i].refclk; i++)
1486                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1487                     table[i].cdclk == cdclk)
1488                         return dev_priv->display.cdclk.hw.ref * table[i].ratio;
1489
1490         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1491                  cdclk, dev_priv->display.cdclk.hw.ref);
1492         return 0;
1493 }
1494
1495 static u8 bxt_calc_voltage_level(int cdclk)
1496 {
1497         return DIV_ROUND_UP(cdclk, 25000);
1498 }
1499
1500 static u8 calc_voltage_level(int cdclk, int num_voltage_levels,
1501                              const int voltage_level_max_cdclk[])
1502 {
1503         int voltage_level;
1504
1505         for (voltage_level = 0; voltage_level < num_voltage_levels; voltage_level++) {
1506                 if (cdclk <= voltage_level_max_cdclk[voltage_level])
1507                         return voltage_level;
1508         }
1509
1510         MISSING_CASE(cdclk);
1511         return num_voltage_levels - 1;
1512 }
1513
1514 static u8 icl_calc_voltage_level(int cdclk)
1515 {
1516         static const int icl_voltage_level_max_cdclk[] = {
1517                 [0] = 312000,
1518                 [1] = 556800,
1519                 [2] = 652800,
1520         };
1521
1522         return calc_voltage_level(cdclk,
1523                                   ARRAY_SIZE(icl_voltage_level_max_cdclk),
1524                                   icl_voltage_level_max_cdclk);
1525 }
1526
1527 static u8 ehl_calc_voltage_level(int cdclk)
1528 {
1529         static const int ehl_voltage_level_max_cdclk[] = {
1530                 [0] = 180000,
1531                 [1] = 312000,
1532                 [2] = 326400,
1533                 /*
1534                  * Bspec lists the limit as 556.8 MHz, but some JSL
1535                  * development boards (at least) boot with 652.8 MHz
1536                  */
1537                 [3] = 652800,
1538         };
1539
1540         return calc_voltage_level(cdclk,
1541                                   ARRAY_SIZE(ehl_voltage_level_max_cdclk),
1542                                   ehl_voltage_level_max_cdclk);
1543 }
1544
1545 static u8 tgl_calc_voltage_level(int cdclk)
1546 {
1547         static const int tgl_voltage_level_max_cdclk[] = {
1548                 [0] = 312000,
1549                 [1] = 326400,
1550                 [2] = 556800,
1551                 [3] = 652800,
1552         };
1553
1554         return calc_voltage_level(cdclk,
1555                                   ARRAY_SIZE(tgl_voltage_level_max_cdclk),
1556                                   tgl_voltage_level_max_cdclk);
1557 }
1558
1559 static u8 rplu_calc_voltage_level(int cdclk)
1560 {
1561         static const int rplu_voltage_level_max_cdclk[] = {
1562                 [0] = 312000,
1563                 [1] = 480000,
1564                 [2] = 556800,
1565                 [3] = 652800,
1566         };
1567
1568         return calc_voltage_level(cdclk,
1569                                   ARRAY_SIZE(rplu_voltage_level_max_cdclk),
1570                                   rplu_voltage_level_max_cdclk);
1571 }
1572
1573 static void icl_readout_refclk(struct drm_i915_private *dev_priv,
1574                                struct intel_cdclk_config *cdclk_config)
1575 {
1576         u32 dssm = intel_de_read(dev_priv, SKL_DSSM) & ICL_DSSM_CDCLK_PLL_REFCLK_MASK;
1577
1578         switch (dssm) {
1579         default:
1580                 MISSING_CASE(dssm);
1581                 fallthrough;
1582         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
1583                 cdclk_config->ref = 24000;
1584                 break;
1585         case ICL_DSSM_CDCLK_PLL_REFCLK_19_2MHz:
1586                 cdclk_config->ref = 19200;
1587                 break;
1588         case ICL_DSSM_CDCLK_PLL_REFCLK_38_4MHz:
1589                 cdclk_config->ref = 38400;
1590                 break;
1591         }
1592 }
1593
1594 static void bxt_de_pll_readout(struct drm_i915_private *dev_priv,
1595                                struct intel_cdclk_config *cdclk_config)
1596 {
1597         u32 val, ratio;
1598
1599         if (IS_DG2(dev_priv))
1600                 cdclk_config->ref = 38400;
1601         else if (DISPLAY_VER(dev_priv) >= 11)
1602                 icl_readout_refclk(dev_priv, cdclk_config);
1603         else
1604                 cdclk_config->ref = 19200;
1605
1606         val = intel_de_read(dev_priv, BXT_DE_PLL_ENABLE);
1607         if ((val & BXT_DE_PLL_PLL_ENABLE) == 0 ||
1608             (val & BXT_DE_PLL_LOCK) == 0) {
1609                 /*
1610                  * CDCLK PLL is disabled, the VCO/ratio doesn't matter, but
1611                  * setting it to zero is a way to signal that.
1612                  */
1613                 cdclk_config->vco = 0;
1614                 return;
1615         }
1616
1617         /*
1618          * DISPLAY_VER >= 11 have the ratio directly in the PLL enable register,
1619          * gen9lp had it in a separate PLL control register.
1620          */
1621         if (DISPLAY_VER(dev_priv) >= 11)
1622                 ratio = val & ICL_CDCLK_PLL_RATIO_MASK;
1623         else
1624                 ratio = intel_de_read(dev_priv, BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
1625
1626         cdclk_config->vco = ratio * cdclk_config->ref;
1627 }
1628
1629 static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
1630                           struct intel_cdclk_config *cdclk_config)
1631 {
1632         u32 squash_ctl = 0;
1633         u32 divider;
1634         int div;
1635
1636         bxt_de_pll_readout(dev_priv, cdclk_config);
1637
1638         if (DISPLAY_VER(dev_priv) >= 12)
1639                 cdclk_config->bypass = cdclk_config->ref / 2;
1640         else if (DISPLAY_VER(dev_priv) >= 11)
1641                 cdclk_config->bypass = 50000;
1642         else
1643                 cdclk_config->bypass = cdclk_config->ref;
1644
1645         if (cdclk_config->vco == 0) {
1646                 cdclk_config->cdclk = cdclk_config->bypass;
1647                 goto out;
1648         }
1649
1650         divider = intel_de_read(dev_priv, CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
1651
1652         switch (divider) {
1653         case BXT_CDCLK_CD2X_DIV_SEL_1:
1654                 div = 2;
1655                 break;
1656         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
1657                 div = 3;
1658                 break;
1659         case BXT_CDCLK_CD2X_DIV_SEL_2:
1660                 div = 4;
1661                 break;
1662         case BXT_CDCLK_CD2X_DIV_SEL_4:
1663                 div = 8;
1664                 break;
1665         default:
1666                 MISSING_CASE(divider);
1667                 return;
1668         }
1669
1670         if (HAS_CDCLK_SQUASH(dev_priv))
1671                 squash_ctl = intel_de_read(dev_priv, CDCLK_SQUASH_CTL);
1672
1673         if (squash_ctl & CDCLK_SQUASH_ENABLE) {
1674                 u16 waveform;
1675                 int size;
1676
1677                 size = REG_FIELD_GET(CDCLK_SQUASH_WINDOW_SIZE_MASK, squash_ctl) + 1;
1678                 waveform = REG_FIELD_GET(CDCLK_SQUASH_WAVEFORM_MASK, squash_ctl) >> (16 - size);
1679
1680                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(hweight16(waveform) *
1681                                                         cdclk_config->vco, size * div);
1682         } else {
1683                 cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_config->vco, div);
1684         }
1685
1686  out:
1687         if (DISPLAY_VER(dev_priv) >= 20)
1688                 cdclk_config->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
1689         /*
1690          * Can't read this out :( Let's assume it's
1691          * at least what the CDCLK frequency requires.
1692          */
1693         cdclk_config->voltage_level =
1694                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config->cdclk);
1695 }
1696
1697 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
1698 {
1699         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, 0);
1700
1701         /* Timeout 200us */
1702         if (intel_de_wait_for_clear(dev_priv,
1703                                     BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1704                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
1705
1706         dev_priv->display.cdclk.hw.vco = 0;
1707 }
1708
1709 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
1710 {
1711         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1712
1713         intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
1714                      BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
1715
1716         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
1717
1718         /* Timeout 200us */
1719         if (intel_de_wait_for_set(dev_priv,
1720                                   BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1721                 drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
1722
1723         dev_priv->display.cdclk.hw.vco = vco;
1724 }
1725
1726 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
1727 {
1728         intel_de_rmw(dev_priv, BXT_DE_PLL_ENABLE,
1729                      BXT_DE_PLL_PLL_ENABLE, 0);
1730
1731         /* Timeout 200us */
1732         if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1733                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
1734
1735         dev_priv->display.cdclk.hw.vco = 0;
1736 }
1737
1738 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
1739 {
1740         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1741         u32 val;
1742
1743         val = ICL_CDCLK_PLL_RATIO(ratio);
1744         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1745
1746         val |= BXT_DE_PLL_PLL_ENABLE;
1747         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1748
1749         /* Timeout 200us */
1750         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
1751                 drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
1752
1753         dev_priv->display.cdclk.hw.vco = vco;
1754 }
1755
1756 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
1757 {
1758         int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
1759         u32 val;
1760
1761         /* Write PLL ratio without disabling */
1762         val = ICL_CDCLK_PLL_RATIO(ratio) | BXT_DE_PLL_PLL_ENABLE;
1763         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1764
1765         /* Submit freq change request */
1766         val |= BXT_DE_PLL_FREQ_REQ;
1767         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1768
1769         /* Timeout 200us */
1770         if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE,
1771                                   BXT_DE_PLL_LOCK | BXT_DE_PLL_FREQ_REQ_ACK, 1))
1772                 drm_err(&dev_priv->drm, "timeout waiting for FREQ change request ack\n");
1773
1774         val &= ~BXT_DE_PLL_FREQ_REQ;
1775         intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
1776
1777         dev_priv->display.cdclk.hw.vco = vco;
1778 }
1779
1780 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
1781 {
1782         if (DISPLAY_VER(dev_priv) >= 12) {
1783                 if (pipe == INVALID_PIPE)
1784                         return TGL_CDCLK_CD2X_PIPE_NONE;
1785                 else
1786                         return TGL_CDCLK_CD2X_PIPE(pipe);
1787         } else if (DISPLAY_VER(dev_priv) >= 11) {
1788                 if (pipe == INVALID_PIPE)
1789                         return ICL_CDCLK_CD2X_PIPE_NONE;
1790                 else
1791                         return ICL_CDCLK_CD2X_PIPE(pipe);
1792         } else {
1793                 if (pipe == INVALID_PIPE)
1794                         return BXT_CDCLK_CD2X_PIPE_NONE;
1795                 else
1796                         return BXT_CDCLK_CD2X_PIPE(pipe);
1797         }
1798 }
1799
1800 static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
1801                                   int cdclk, int vco, u16 waveform)
1802 {
1803         /* cdclk = vco / 2 / div{1,1.5,2,4} */
1804         switch (cdclk_divider(cdclk, vco, waveform)) {
1805         default:
1806                 drm_WARN_ON(&dev_priv->drm,
1807                             cdclk != dev_priv->display.cdclk.hw.bypass);
1808                 drm_WARN_ON(&dev_priv->drm, vco != 0);
1809                 fallthrough;
1810         case 2:
1811                 return BXT_CDCLK_CD2X_DIV_SEL_1;
1812         case 3:
1813                 return BXT_CDCLK_CD2X_DIV_SEL_1_5;
1814         case 4:
1815                 return BXT_CDCLK_CD2X_DIV_SEL_2;
1816         case 8:
1817                 return BXT_CDCLK_CD2X_DIV_SEL_4;
1818         }
1819 }
1820
1821 static u16 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
1822                                  int cdclk)
1823 {
1824         const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
1825         int i;
1826
1827         if (cdclk == dev_priv->display.cdclk.hw.bypass)
1828                 return 0;
1829
1830         for (i = 0; table[i].refclk; i++)
1831                 if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
1832                     table[i].cdclk == cdclk)
1833                         return table[i].waveform;
1834
1835         drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
1836                  cdclk, dev_priv->display.cdclk.hw.ref);
1837
1838         return 0xffff;
1839 }
1840
1841 static void icl_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1842 {
1843         if (i915->display.cdclk.hw.vco != 0 &&
1844             i915->display.cdclk.hw.vco != vco)
1845                 icl_cdclk_pll_disable(i915);
1846
1847         if (i915->display.cdclk.hw.vco != vco)
1848                 icl_cdclk_pll_enable(i915, vco);
1849 }
1850
1851 static void bxt_cdclk_pll_update(struct drm_i915_private *i915, int vco)
1852 {
1853         if (i915->display.cdclk.hw.vco != 0 &&
1854             i915->display.cdclk.hw.vco != vco)
1855                 bxt_de_pll_disable(i915);
1856
1857         if (i915->display.cdclk.hw.vco != vco)
1858                 bxt_de_pll_enable(i915, vco);
1859 }
1860
1861 static void dg2_cdclk_squash_program(struct drm_i915_private *i915,
1862                                      u16 waveform)
1863 {
1864         u32 squash_ctl = 0;
1865
1866         if (waveform)
1867                 squash_ctl = CDCLK_SQUASH_ENABLE |
1868                              CDCLK_SQUASH_WINDOW_SIZE(0xf) | waveform;
1869
1870         intel_de_write(i915, CDCLK_SQUASH_CTL, squash_ctl);
1871 }
1872
1873 static bool cdclk_pll_is_unknown(unsigned int vco)
1874 {
1875         /*
1876          * Ensure driver does not take the crawl path for the
1877          * case when the vco is set to ~0 in the
1878          * sanitize path.
1879          */
1880         return vco == ~0;
1881 }
1882
1883 static bool mdclk_source_is_cdclk_pll(struct drm_i915_private *i915)
1884 {
1885         return DISPLAY_VER(i915) >= 20;
1886 }
1887
1888 static u32 xe2lpd_mdclk_source_sel(struct drm_i915_private *i915)
1889 {
1890         if (mdclk_source_is_cdclk_pll(i915))
1891                 return MDCLK_SOURCE_SEL_CDCLK_PLL;
1892
1893         return MDCLK_SOURCE_SEL_CD2XCLK;
1894 }
1895
1896 int intel_mdclk_cdclk_ratio(struct drm_i915_private *i915,
1897                             const struct intel_cdclk_config *cdclk_config)
1898 {
1899         if (mdclk_source_is_cdclk_pll(i915))
1900                 return DIV_ROUND_UP(cdclk_config->vco, cdclk_config->cdclk);
1901
1902         /* Otherwise, source for MDCLK is CD2XCLK. */
1903         return 2;
1904 }
1905
1906 static void xe2lpd_mdclk_cdclk_ratio_program(struct drm_i915_private *i915,
1907                                              const struct intel_cdclk_config *cdclk_config)
1908 {
1909         intel_dbuf_mdclk_cdclk_ratio_update(i915,
1910                                             intel_mdclk_cdclk_ratio(i915, cdclk_config),
1911                                             cdclk_config->joined_mbus);
1912 }
1913
1914 static bool cdclk_compute_crawl_and_squash_midpoint(struct drm_i915_private *i915,
1915                                                     const struct intel_cdclk_config *old_cdclk_config,
1916                                                     const struct intel_cdclk_config *new_cdclk_config,
1917                                                     struct intel_cdclk_config *mid_cdclk_config)
1918 {
1919         u16 old_waveform, new_waveform, mid_waveform;
1920         int old_div, new_div, mid_div;
1921
1922         /* Return if PLL is in an unknown state, force a complete disable and re-enable. */
1923         if (cdclk_pll_is_unknown(old_cdclk_config->vco))
1924                 return false;
1925
1926         /* Return if both Squash and Crawl are not present */
1927         if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
1928                 return false;
1929
1930         old_waveform = cdclk_squash_waveform(i915, old_cdclk_config->cdclk);
1931         new_waveform = cdclk_squash_waveform(i915, new_cdclk_config->cdclk);
1932
1933         /* Return if Squash only or Crawl only is the desired action */
1934         if (old_cdclk_config->vco == 0 || new_cdclk_config->vco == 0 ||
1935             old_cdclk_config->vco == new_cdclk_config->vco ||
1936             old_waveform == new_waveform)
1937                 return false;
1938
1939         old_div = cdclk_divider(old_cdclk_config->cdclk,
1940                                 old_cdclk_config->vco, old_waveform);
1941         new_div = cdclk_divider(new_cdclk_config->cdclk,
1942                                 new_cdclk_config->vco, new_waveform);
1943
1944         /*
1945          * Should not happen currently. We might need more midpoint
1946          * transitions if we need to also change the cd2x divider.
1947          */
1948         if (drm_WARN_ON(&i915->drm, old_div != new_div))
1949                 return false;
1950
1951         *mid_cdclk_config = *new_cdclk_config;
1952
1953         /*
1954          * Populate the mid_cdclk_config accordingly.
1955          * - If moving to a higher cdclk, the desired action is squashing.
1956          * The mid cdclk config should have the new (squash) waveform.
1957          * - If moving to a lower cdclk, the desired action is crawling.
1958          * The mid cdclk config should have the new vco.
1959          */
1960
1961         if (cdclk_squash_divider(new_waveform) > cdclk_squash_divider(old_waveform)) {
1962                 mid_cdclk_config->vco = old_cdclk_config->vco;
1963                 mid_div = old_div;
1964                 mid_waveform = new_waveform;
1965         } else {
1966                 mid_cdclk_config->vco = new_cdclk_config->vco;
1967                 mid_div = new_div;
1968                 mid_waveform = old_waveform;
1969         }
1970
1971         mid_cdclk_config->cdclk = DIV_ROUND_CLOSEST(cdclk_squash_divider(mid_waveform) *
1972                                                     mid_cdclk_config->vco,
1973                                                     cdclk_squash_len * mid_div);
1974
1975         /* make sure the mid clock came out sane */
1976
1977         drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk <
1978                     min(old_cdclk_config->cdclk, new_cdclk_config->cdclk));
1979         drm_WARN_ON(&i915->drm, mid_cdclk_config->cdclk >
1980                     i915->display.cdclk.max_cdclk_freq);
1981         drm_WARN_ON(&i915->drm, cdclk_squash_waveform(i915, mid_cdclk_config->cdclk) !=
1982                     mid_waveform);
1983
1984         return true;
1985 }
1986
1987 static bool pll_enable_wa_needed(struct drm_i915_private *dev_priv)
1988 {
1989         return (DISPLAY_VER_FULL(dev_priv) == IP_VER(20, 0) ||
1990                 DISPLAY_VER_FULL(dev_priv) == IP_VER(14, 0) ||
1991                 IS_DG2(dev_priv)) &&
1992                 dev_priv->display.cdclk.hw.vco > 0;
1993 }
1994
1995 static u32 bxt_cdclk_ctl(struct drm_i915_private *i915,
1996                          const struct intel_cdclk_config *cdclk_config,
1997                          enum pipe pipe)
1998 {
1999         int cdclk = cdclk_config->cdclk;
2000         int vco = cdclk_config->vco;
2001         u16 waveform;
2002         u32 val;
2003
2004         waveform = cdclk_squash_waveform(i915, cdclk);
2005
2006         val = bxt_cdclk_cd2x_div_sel(i915, cdclk, vco, waveform) |
2007                 bxt_cdclk_cd2x_pipe(i915, pipe);
2008
2009         /*
2010          * Disable SSA Precharge when CD clock frequency < 500 MHz,
2011          * enable otherwise.
2012          */
2013         if ((IS_GEMINILAKE(i915) || IS_BROXTON(i915)) &&
2014             cdclk >= 500000)
2015                 val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
2016
2017         if (DISPLAY_VER(i915) >= 20)
2018                 val |= xe2lpd_mdclk_source_sel(i915);
2019         else
2020                 val |= skl_cdclk_decimal(cdclk);
2021
2022         return val;
2023 }
2024
2025 static void _bxt_set_cdclk(struct drm_i915_private *dev_priv,
2026                            const struct intel_cdclk_config *cdclk_config,
2027                            enum pipe pipe)
2028 {
2029         int cdclk = cdclk_config->cdclk;
2030         int vco = cdclk_config->vco;
2031
2032         if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0 &&
2033             !cdclk_pll_is_unknown(dev_priv->display.cdclk.hw.vco)) {
2034                 if (dev_priv->display.cdclk.hw.vco != vco)
2035                         adlp_cdclk_pll_crawl(dev_priv, vco);
2036         } else if (DISPLAY_VER(dev_priv) >= 11) {
2037                 /* wa_15010685871: dg2, mtl */
2038                 if (pll_enable_wa_needed(dev_priv))
2039                         dg2_cdclk_squash_program(dev_priv, 0);
2040
2041                 icl_cdclk_pll_update(dev_priv, vco);
2042         } else
2043                 bxt_cdclk_pll_update(dev_priv, vco);
2044
2045         if (HAS_CDCLK_SQUASH(dev_priv)) {
2046                 u16 waveform = cdclk_squash_waveform(dev_priv, cdclk);
2047
2048                 dg2_cdclk_squash_program(dev_priv, waveform);
2049         }
2050
2051         intel_de_write(dev_priv, CDCLK_CTL, bxt_cdclk_ctl(dev_priv, cdclk_config, pipe));
2052
2053         if (pipe != INVALID_PIPE)
2054                 intel_crtc_wait_for_next_vblank(intel_crtc_for_pipe(dev_priv, pipe));
2055 }
2056
2057 static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
2058                           const struct intel_cdclk_config *cdclk_config,
2059                           enum pipe pipe)
2060 {
2061         struct intel_cdclk_config mid_cdclk_config;
2062         int cdclk = cdclk_config->cdclk;
2063         int ret = 0;
2064
2065         /*
2066          * Inform power controller of upcoming frequency change.
2067          * Display versions 14 and beyond do not follow the PUnit
2068          * mailbox communication, skip
2069          * this step.
2070          */
2071         if (DISPLAY_VER(dev_priv) >= 14 || IS_DG2(dev_priv))
2072                 /* NOOP */;
2073         else if (DISPLAY_VER(dev_priv) >= 11)
2074                 ret = skl_pcode_request(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
2075                                         SKL_CDCLK_PREPARE_FOR_CHANGE,
2076                                         SKL_CDCLK_READY_FOR_CHANGE,
2077                                         SKL_CDCLK_READY_FOR_CHANGE, 3);
2078         else
2079                 /*
2080                  * BSpec requires us to wait up to 150usec, but that leads to
2081                  * timeouts; the 2ms used here is based on experiment.
2082                  */
2083                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
2084                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
2085                                               0x80000000, 150, 2);
2086
2087         if (ret) {
2088                 drm_err(&dev_priv->drm,
2089                         "Failed to inform PCU about cdclk change (err %d, freq %d)\n",
2090                         ret, cdclk);
2091                 return;
2092         }
2093
2094         if (DISPLAY_VER(dev_priv) >= 20 && cdclk < dev_priv->display.cdclk.hw.cdclk)
2095                 xe2lpd_mdclk_cdclk_ratio_program(dev_priv, cdclk_config);
2096
2097         if (cdclk_compute_crawl_and_squash_midpoint(dev_priv, &dev_priv->display.cdclk.hw,
2098                                                     cdclk_config, &mid_cdclk_config)) {
2099                 _bxt_set_cdclk(dev_priv, &mid_cdclk_config, pipe);
2100                 _bxt_set_cdclk(dev_priv, cdclk_config, pipe);
2101         } else {
2102                 _bxt_set_cdclk(dev_priv, cdclk_config, pipe);
2103         }
2104
2105         if (DISPLAY_VER(dev_priv) >= 20 && cdclk > dev_priv->display.cdclk.hw.cdclk)
2106                 xe2lpd_mdclk_cdclk_ratio_program(dev_priv, cdclk_config);
2107
2108         if (DISPLAY_VER(dev_priv) >= 14)
2109                 /*
2110                  * NOOP - No Pcode communication needed for
2111                  * Display versions 14 and beyond
2112                  */;
2113         else if (DISPLAY_VER(dev_priv) >= 11 && !IS_DG2(dev_priv))
2114                 ret = snb_pcode_write(&dev_priv->uncore, SKL_PCODE_CDCLK_CONTROL,
2115                                       cdclk_config->voltage_level);
2116         if (DISPLAY_VER(dev_priv) < 11) {
2117                 /*
2118                  * The timeout isn't specified, the 2ms used here is based on
2119                  * experiment.
2120                  * FIXME: Waiting for the request completion could be delayed
2121                  * until the next PCODE request based on BSpec.
2122                  */
2123                 ret = snb_pcode_write_timeout(&dev_priv->uncore,
2124                                               HSW_PCODE_DE_WRITE_FREQ_REQ,
2125                                               cdclk_config->voltage_level,
2126                                               150, 2);
2127         }
2128         if (ret) {
2129                 drm_err(&dev_priv->drm,
2130                         "PCode CDCLK freq set failed, (err %d, freq %d)\n",
2131                         ret, cdclk);
2132                 return;
2133         }
2134
2135         intel_update_cdclk(dev_priv);
2136
2137         if (DISPLAY_VER(dev_priv) >= 11)
2138                 /*
2139                  * Can't read out the voltage level :(
2140                  * Let's just assume everything is as expected.
2141                  */
2142                 dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level;
2143 }
2144
2145 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
2146 {
2147         u32 cdctl, expected;
2148         int cdclk, vco;
2149
2150         intel_update_cdclk(dev_priv);
2151         intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
2152
2153         if (dev_priv->display.cdclk.hw.vco == 0 ||
2154             dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
2155                 goto sanitize;
2156
2157         /* Make sure this is a legal cdclk value for the platform */
2158         cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk);
2159         if (cdclk != dev_priv->display.cdclk.hw.cdclk)
2160                 goto sanitize;
2161
2162         /* Make sure the VCO is correct for the cdclk */
2163         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
2164         if (vco != dev_priv->display.cdclk.hw.vco)
2165                 goto sanitize;
2166
2167         /*
2168          * Some BIOS versions leave an incorrect decimal frequency value and
2169          * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
2170          * so sanitize this register.
2171          */
2172         cdctl = intel_de_read(dev_priv, CDCLK_CTL);
2173         expected = bxt_cdclk_ctl(dev_priv, &dev_priv->display.cdclk.hw, INVALID_PIPE);
2174
2175         /*
2176          * Let's ignore the pipe field, since BIOS could have configured the
2177          * dividers both synching to an active pipe, or asynchronously
2178          * (PIPE_NONE).
2179          */
2180         cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
2181         expected &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
2182
2183         if (cdctl == expected)
2184                 /* All well; nothing to sanitize */
2185                 return;
2186
2187 sanitize:
2188         drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
2189
2190         /* force cdclk programming */
2191         dev_priv->display.cdclk.hw.cdclk = 0;
2192
2193         /* force full PLL disable + enable */
2194         dev_priv->display.cdclk.hw.vco = ~0;
2195 }
2196
2197 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
2198 {
2199         struct intel_cdclk_config cdclk_config;
2200
2201         bxt_sanitize_cdclk(dev_priv);
2202
2203         if (dev_priv->display.cdclk.hw.cdclk != 0 &&
2204             dev_priv->display.cdclk.hw.vco != 0)
2205                 return;
2206
2207         cdclk_config = dev_priv->display.cdclk.hw;
2208
2209         /*
2210          * FIXME:
2211          * - The initial CDCLK needs to be read from VBT.
2212          *   Need to make this change after VBT has changes for BXT.
2213          */
2214         cdclk_config.cdclk = bxt_calc_cdclk(dev_priv, 0);
2215         cdclk_config.vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk_config.cdclk);
2216         cdclk_config.voltage_level =
2217                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2218
2219         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2220 }
2221
2222 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
2223 {
2224         struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
2225
2226         cdclk_config.cdclk = cdclk_config.bypass;
2227         cdclk_config.vco = 0;
2228         cdclk_config.voltage_level =
2229                 intel_cdclk_calc_voltage_level(dev_priv, cdclk_config.cdclk);
2230
2231         bxt_set_cdclk(dev_priv, &cdclk_config, INVALID_PIPE);
2232 }
2233
2234 /**
2235  * intel_cdclk_init_hw - Initialize CDCLK hardware
2236  * @i915: i915 device
2237  *
2238  * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and
2239  * sanitizing the state of the hardware if needed. This is generally done only
2240  * during the display core initialization sequence, after which the DMC will
2241  * take care of turning CDCLK off/on as needed.
2242  */
2243 void intel_cdclk_init_hw(struct drm_i915_private *i915)
2244 {
2245         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2246                 bxt_cdclk_init_hw(i915);
2247         else if (DISPLAY_VER(i915) == 9)
2248                 skl_cdclk_init_hw(i915);
2249 }
2250
2251 /**
2252  * intel_cdclk_uninit_hw - Uninitialize CDCLK hardware
2253  * @i915: i915 device
2254  *
2255  * Uninitialize CDCLK. This is done only during the display core
2256  * uninitialization sequence.
2257  */
2258 void intel_cdclk_uninit_hw(struct drm_i915_private *i915)
2259 {
2260         if (DISPLAY_VER(i915) >= 10 || IS_BROXTON(i915))
2261                 bxt_cdclk_uninit_hw(i915);
2262         else if (DISPLAY_VER(i915) == 9)
2263                 skl_cdclk_uninit_hw(i915);
2264 }
2265
2266 static bool intel_cdclk_can_crawl_and_squash(struct drm_i915_private *i915,
2267                                              const struct intel_cdclk_config *a,
2268                                              const struct intel_cdclk_config *b)
2269 {
2270         u16 old_waveform;
2271         u16 new_waveform;
2272
2273         drm_WARN_ON(&i915->drm, cdclk_pll_is_unknown(a->vco));
2274
2275         if (a->vco == 0 || b->vco == 0)
2276                 return false;
2277
2278         if (!HAS_CDCLK_CRAWL(i915) || !HAS_CDCLK_SQUASH(i915))
2279                 return false;
2280
2281         old_waveform = cdclk_squash_waveform(i915, a->cdclk);
2282         new_waveform = cdclk_squash_waveform(i915, b->cdclk);
2283
2284         return a->vco != b->vco &&
2285                old_waveform != new_waveform;
2286 }
2287
2288 static bool intel_cdclk_can_crawl(struct drm_i915_private *dev_priv,
2289                                   const struct intel_cdclk_config *a,
2290                                   const struct intel_cdclk_config *b)
2291 {
2292         int a_div, b_div;
2293
2294         if (!HAS_CDCLK_CRAWL(dev_priv))
2295                 return false;
2296
2297         /*
2298          * The vco and cd2x divider will change independently
2299          * from each, so we disallow cd2x change when crawling.
2300          */
2301         a_div = DIV_ROUND_CLOSEST(a->vco, a->cdclk);
2302         b_div = DIV_ROUND_CLOSEST(b->vco, b->cdclk);
2303
2304         return a->vco != 0 && b->vco != 0 &&
2305                 a->vco != b->vco &&
2306                 a_div == b_div &&
2307                 a->ref == b->ref;
2308 }
2309
2310 static bool intel_cdclk_can_squash(struct drm_i915_private *dev_priv,
2311                                    const struct intel_cdclk_config *a,
2312                                    const struct intel_cdclk_config *b)
2313 {
2314         /*
2315          * FIXME should store a bit more state in intel_cdclk_config
2316          * to differentiate squasher vs. cd2x divider properly. For
2317          * the moment all platforms with squasher use a fixed cd2x
2318          * divider.
2319          */
2320         if (!HAS_CDCLK_SQUASH(dev_priv))
2321                 return false;
2322
2323         return a->cdclk != b->cdclk &&
2324                 a->vco != 0 &&
2325                 a->vco == b->vco &&
2326                 a->ref == b->ref;
2327 }
2328
2329 /**
2330  * intel_cdclk_clock_changed - Check whether the clock changed
2331  * @a: first CDCLK configuration
2332  * @b: second CDCLK configuration
2333  *
2334  * Returns:
2335  * True if CDCLK changed in a way that requires re-programming and
2336  * False otherwise.
2337  */
2338 bool intel_cdclk_clock_changed(const struct intel_cdclk_config *a,
2339                                const struct intel_cdclk_config *b)
2340 {
2341         return a->cdclk != b->cdclk ||
2342                 a->vco != b->vco ||
2343                 a->ref != b->ref;
2344 }
2345
2346 /**
2347  * intel_cdclk_can_cd2x_update - Determine if changing between the two CDCLK
2348  *                               configurations requires only a cd2x divider update
2349  * @dev_priv: i915 device
2350  * @a: first CDCLK configuration
2351  * @b: second CDCLK configuration
2352  *
2353  * Returns:
2354  * True if changing between the two CDCLK configurations
2355  * can be done with just a cd2x divider update, false if not.
2356  */
2357 static bool intel_cdclk_can_cd2x_update(struct drm_i915_private *dev_priv,
2358                                         const struct intel_cdclk_config *a,
2359                                         const struct intel_cdclk_config *b)
2360 {
2361         /* Older hw doesn't have the capability */
2362         if (DISPLAY_VER(dev_priv) < 10 && !IS_BROXTON(dev_priv))
2363                 return false;
2364
2365         /*
2366          * FIXME should store a bit more state in intel_cdclk_config
2367          * to differentiate squasher vs. cd2x divider properly. For
2368          * the moment all platforms with squasher use a fixed cd2x
2369          * divider.
2370          */
2371         if (HAS_CDCLK_SQUASH(dev_priv))
2372                 return false;
2373
2374         return a->cdclk != b->cdclk &&
2375                 a->vco != 0 &&
2376                 a->vco == b->vco &&
2377                 a->ref == b->ref;
2378 }
2379
2380 /**
2381  * intel_cdclk_changed - Determine if two CDCLK configurations are different
2382  * @a: first CDCLK configuration
2383  * @b: second CDCLK configuration
2384  *
2385  * Returns:
2386  * True if the CDCLK configurations don't match, false if they do.
2387  */
2388 static bool intel_cdclk_changed(const struct intel_cdclk_config *a,
2389                                 const struct intel_cdclk_config *b)
2390 {
2391         return intel_cdclk_clock_changed(a, b) ||
2392                 a->voltage_level != b->voltage_level;
2393 }
2394
2395 void intel_cdclk_dump_config(struct drm_i915_private *i915,
2396                              const struct intel_cdclk_config *cdclk_config,
2397                              const char *context)
2398 {
2399         drm_dbg_kms(&i915->drm, "%s %d kHz, VCO %d kHz, ref %d kHz, bypass %d kHz, voltage level %d\n",
2400                     context, cdclk_config->cdclk, cdclk_config->vco,
2401                     cdclk_config->ref, cdclk_config->bypass,
2402                     cdclk_config->voltage_level);
2403 }
2404
2405 static void intel_pcode_notify(struct drm_i915_private *i915,
2406                                u8 voltage_level,
2407                                u8 active_pipe_count,
2408                                u16 cdclk,
2409                                bool cdclk_update_valid,
2410                                bool pipe_count_update_valid)
2411 {
2412         int ret;
2413         u32 update_mask = 0;
2414
2415         if (!IS_DG2(i915))
2416                 return;
2417
2418         update_mask = DISPLAY_TO_PCODE_UPDATE_MASK(cdclk, active_pipe_count, voltage_level);
2419
2420         if (cdclk_update_valid)
2421                 update_mask |= DISPLAY_TO_PCODE_CDCLK_VALID;
2422
2423         if (pipe_count_update_valid)
2424                 update_mask |= DISPLAY_TO_PCODE_PIPE_COUNT_VALID;
2425
2426         ret = skl_pcode_request(&i915->uncore, SKL_PCODE_CDCLK_CONTROL,
2427                                 SKL_CDCLK_PREPARE_FOR_CHANGE |
2428                                 update_mask,
2429                                 SKL_CDCLK_READY_FOR_CHANGE,
2430                                 SKL_CDCLK_READY_FOR_CHANGE, 3);
2431         if (ret)
2432                 drm_err(&i915->drm,
2433                         "Failed to inform PCU about display config (err %d)\n",
2434                         ret);
2435 }
2436
2437 static void intel_set_cdclk(struct drm_i915_private *dev_priv,
2438                             const struct intel_cdclk_config *cdclk_config,
2439                             enum pipe pipe, const char *context)
2440 {
2441         struct intel_encoder *encoder;
2442
2443         if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config))
2444                 return;
2445
2446         if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
2447                 return;
2448
2449         intel_cdclk_dump_config(dev_priv, cdclk_config, context);
2450
2451         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2452                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2453
2454                 intel_psr_pause(intel_dp);
2455         }
2456
2457         intel_audio_cdclk_change_pre(dev_priv);
2458
2459         /*
2460          * Lock aux/gmbus while we change cdclk in case those
2461          * functions use cdclk. Not all platforms/ports do,
2462          * but we'll lock them all for simplicity.
2463          */
2464         mutex_lock(&dev_priv->display.gmbus.mutex);
2465         for_each_intel_dp(&dev_priv->drm, encoder) {
2466                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2467
2468                 mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
2469                                      &dev_priv->display.gmbus.mutex);
2470         }
2471
2472         intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
2473
2474         for_each_intel_dp(&dev_priv->drm, encoder) {
2475                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2476
2477                 mutex_unlock(&intel_dp->aux.hw_mutex);
2478         }
2479         mutex_unlock(&dev_priv->display.gmbus.mutex);
2480
2481         for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
2482                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
2483
2484                 intel_psr_resume(intel_dp);
2485         }
2486
2487         intel_audio_cdclk_change_post(dev_priv);
2488
2489         if (drm_WARN(&dev_priv->drm,
2490                      intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config),
2491                      "cdclk state doesn't match!\n")) {
2492                 intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]");
2493                 intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
2494         }
2495 }
2496
2497 static void intel_cdclk_pcode_pre_notify(struct intel_atomic_state *state)
2498 {
2499         struct drm_i915_private *i915 = to_i915(state->base.dev);
2500         const struct intel_cdclk_state *old_cdclk_state =
2501                 intel_atomic_get_old_cdclk_state(state);
2502         const struct intel_cdclk_state *new_cdclk_state =
2503                 intel_atomic_get_new_cdclk_state(state);
2504         unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2505         bool change_cdclk, update_pipe_count;
2506
2507         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2508                                  &new_cdclk_state->actual) &&
2509                                  new_cdclk_state->active_pipes ==
2510                                  old_cdclk_state->active_pipes)
2511                 return;
2512
2513         /* According to "Sequence Before Frequency Change", voltage level set to 0x3 */
2514         voltage_level = DISPLAY_TO_PCODE_VOLTAGE_MAX;
2515
2516         change_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2517         update_pipe_count = hweight8(new_cdclk_state->active_pipes) >
2518                             hweight8(old_cdclk_state->active_pipes);
2519
2520         /*
2521          * According to "Sequence Before Frequency Change",
2522          * if CDCLK is increasing, set bits 25:16 to upcoming CDCLK,
2523          * if CDCLK is decreasing or not changing, set bits 25:16 to current CDCLK,
2524          * which basically means we choose the maximum of old and new CDCLK, if we know both
2525          */
2526         if (change_cdclk)
2527                 cdclk = max(new_cdclk_state->actual.cdclk, old_cdclk_state->actual.cdclk);
2528
2529         /*
2530          * According to "Sequence For Pipe Count Change",
2531          * if pipe count is increasing, set bits 25:16 to upcoming pipe count
2532          * (power well is enabled)
2533          * no action if it is decreasing, before the change
2534          */
2535         if (update_pipe_count)
2536                 num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2537
2538         intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2539                            change_cdclk, update_pipe_count);
2540 }
2541
2542 static void intel_cdclk_pcode_post_notify(struct intel_atomic_state *state)
2543 {
2544         struct drm_i915_private *i915 = to_i915(state->base.dev);
2545         const struct intel_cdclk_state *new_cdclk_state =
2546                 intel_atomic_get_new_cdclk_state(state);
2547         const struct intel_cdclk_state *old_cdclk_state =
2548                 intel_atomic_get_old_cdclk_state(state);
2549         unsigned int cdclk = 0; u8 voltage_level, num_active_pipes = 0;
2550         bool update_cdclk, update_pipe_count;
2551
2552         /* According to "Sequence After Frequency Change", set voltage to used level */
2553         voltage_level = new_cdclk_state->actual.voltage_level;
2554
2555         update_cdclk = new_cdclk_state->actual.cdclk != old_cdclk_state->actual.cdclk;
2556         update_pipe_count = hweight8(new_cdclk_state->active_pipes) <
2557                             hweight8(old_cdclk_state->active_pipes);
2558
2559         /*
2560          * According to "Sequence After Frequency Change",
2561          * set bits 25:16 to current CDCLK
2562          */
2563         if (update_cdclk)
2564                 cdclk = new_cdclk_state->actual.cdclk;
2565
2566         /*
2567          * According to "Sequence For Pipe Count Change",
2568          * if pipe count is decreasing, set bits 25:16 to current pipe count,
2569          * after the change(power well is disabled)
2570          * no action if it is increasing, after the change
2571          */
2572         if (update_pipe_count)
2573                 num_active_pipes = hweight8(new_cdclk_state->active_pipes);
2574
2575         intel_pcode_notify(i915, voltage_level, num_active_pipes, cdclk,
2576                            update_cdclk, update_pipe_count);
2577 }
2578
2579 bool intel_cdclk_is_decreasing_later(struct intel_atomic_state *state)
2580 {
2581         const struct intel_cdclk_state *old_cdclk_state =
2582                 intel_atomic_get_old_cdclk_state(state);
2583         const struct intel_cdclk_state *new_cdclk_state =
2584                 intel_atomic_get_new_cdclk_state(state);
2585
2586         return new_cdclk_state && !new_cdclk_state->disable_pipes &&
2587                 new_cdclk_state->actual.cdclk < old_cdclk_state->actual.cdclk;
2588 }
2589
2590 /**
2591  * intel_set_cdclk_pre_plane_update - Push the CDCLK state to the hardware
2592  * @state: intel atomic state
2593  *
2594  * Program the hardware before updating the HW plane state based on the
2595  * new CDCLK state, if necessary.
2596  */
2597 void
2598 intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state)
2599 {
2600         struct drm_i915_private *i915 = to_i915(state->base.dev);
2601         const struct intel_cdclk_state *old_cdclk_state =
2602                 intel_atomic_get_old_cdclk_state(state);
2603         const struct intel_cdclk_state *new_cdclk_state =
2604                 intel_atomic_get_new_cdclk_state(state);
2605         struct intel_cdclk_config cdclk_config;
2606         enum pipe pipe;
2607
2608         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2609                                  &new_cdclk_state->actual))
2610                 return;
2611
2612         if (IS_DG2(i915))
2613                 intel_cdclk_pcode_pre_notify(state);
2614
2615         if (new_cdclk_state->disable_pipes) {
2616                 cdclk_config = new_cdclk_state->actual;
2617                 pipe = INVALID_PIPE;
2618         } else {
2619                 if (new_cdclk_state->actual.cdclk >= old_cdclk_state->actual.cdclk) {
2620                         cdclk_config = new_cdclk_state->actual;
2621                         pipe = new_cdclk_state->pipe;
2622                 } else {
2623                         cdclk_config = old_cdclk_state->actual;
2624                         pipe = INVALID_PIPE;
2625                 }
2626
2627                 cdclk_config.voltage_level = max(new_cdclk_state->actual.voltage_level,
2628                                                  old_cdclk_state->actual.voltage_level);
2629         }
2630
2631         /*
2632          * mbus joining will be changed later by
2633          * intel_dbuf_mbus_{pre,post}_ddb_update()
2634          */
2635         cdclk_config.joined_mbus = old_cdclk_state->actual.joined_mbus;
2636
2637         drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2638
2639         intel_set_cdclk(i915, &cdclk_config, pipe,
2640                         "Pre changing CDCLK to");
2641 }
2642
2643 /**
2644  * intel_set_cdclk_post_plane_update - Push the CDCLK state to the hardware
2645  * @state: intel atomic state
2646  *
2647  * Program the hardware after updating the HW plane state based on the
2648  * new CDCLK state, if necessary.
2649  */
2650 void
2651 intel_set_cdclk_post_plane_update(struct intel_atomic_state *state)
2652 {
2653         struct drm_i915_private *i915 = to_i915(state->base.dev);
2654         const struct intel_cdclk_state *old_cdclk_state =
2655                 intel_atomic_get_old_cdclk_state(state);
2656         const struct intel_cdclk_state *new_cdclk_state =
2657                 intel_atomic_get_new_cdclk_state(state);
2658         enum pipe pipe;
2659
2660         if (!intel_cdclk_changed(&old_cdclk_state->actual,
2661                                  &new_cdclk_state->actual))
2662                 return;
2663
2664         if (IS_DG2(i915))
2665                 intel_cdclk_pcode_post_notify(state);
2666
2667         if (!new_cdclk_state->disable_pipes &&
2668             new_cdclk_state->actual.cdclk < old_cdclk_state->actual.cdclk)
2669                 pipe = new_cdclk_state->pipe;
2670         else
2671                 pipe = INVALID_PIPE;
2672
2673         drm_WARN_ON(&i915->drm, !new_cdclk_state->base.changed);
2674
2675         intel_set_cdclk(i915, &new_cdclk_state->actual, pipe,
2676                         "Post changing CDCLK to");
2677 }
2678
2679 static int intel_pixel_rate_to_cdclk(const struct intel_crtc_state *crtc_state)
2680 {
2681         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
2682         int pixel_rate = crtc_state->pixel_rate;
2683
2684         if (DISPLAY_VER(dev_priv) >= 10)
2685                 return DIV_ROUND_UP(pixel_rate, 2);
2686         else if (DISPLAY_VER(dev_priv) == 9 ||
2687                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2688                 return pixel_rate;
2689         else if (IS_CHERRYVIEW(dev_priv))
2690                 return DIV_ROUND_UP(pixel_rate * 100, 95);
2691         else if (crtc_state->double_wide)
2692                 return DIV_ROUND_UP(pixel_rate * 100, 90 * 2);
2693         else
2694                 return DIV_ROUND_UP(pixel_rate * 100, 90);
2695 }
2696
2697 static int intel_planes_min_cdclk(const struct intel_crtc_state *crtc_state)
2698 {
2699         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2700         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2701         struct intel_plane *plane;
2702         int min_cdclk = 0;
2703
2704         for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane)
2705                 min_cdclk = max(crtc_state->min_cdclk[plane->id], min_cdclk);
2706
2707         return min_cdclk;
2708 }
2709
2710 static int intel_vdsc_min_cdclk(const struct intel_crtc_state *crtc_state)
2711 {
2712         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2713         struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2714         int num_vdsc_instances = intel_dsc_get_num_vdsc_instances(crtc_state);
2715         int min_cdclk = 0;
2716
2717         /*
2718          * When we decide to use only one VDSC engine, since
2719          * each VDSC operates with 1 ppc throughput, pixel clock
2720          * cannot be higher than the VDSC clock (cdclk)
2721          * If there 2 VDSC engines, then pixel clock can't be higher than
2722          * VDSC clock(cdclk) * 2 and so on.
2723          */
2724         min_cdclk = max_t(int, min_cdclk,
2725                           DIV_ROUND_UP(crtc_state->pixel_rate, num_vdsc_instances));
2726
2727         if (crtc_state->bigjoiner_pipes) {
2728                 int pixel_clock = intel_dp_mode_to_fec_clock(crtc_state->hw.adjusted_mode.clock);
2729
2730                 /*
2731                  * According to Bigjoiner bw check:
2732                  * compressed_bpp <= PPC * CDCLK * Big joiner Interface bits / Pixel clock
2733                  *
2734                  * We have already computed compressed_bpp, so now compute the min CDCLK that
2735                  * is required to support this compressed_bpp.
2736                  *
2737                  * => CDCLK >= compressed_bpp * Pixel clock / (PPC * Bigjoiner Interface bits)
2738                  *
2739                  * Since PPC = 2 with bigjoiner
2740                  * => CDCLK >= compressed_bpp * Pixel clock  / 2 * Bigjoiner Interface bits
2741                  */
2742                 int bigjoiner_interface_bits = DISPLAY_VER(i915) >= 14 ? 36 : 24;
2743                 int min_cdclk_bj =
2744                         (to_bpp_int_roundup(crtc_state->dsc.compressed_bpp_x16) *
2745                          pixel_clock) / (2 * bigjoiner_interface_bits);
2746
2747                 min_cdclk = max(min_cdclk, min_cdclk_bj);
2748         }
2749
2750         return min_cdclk;
2751 }
2752
2753 int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
2754 {
2755         struct drm_i915_private *dev_priv =
2756                 to_i915(crtc_state->uapi.crtc->dev);
2757         int min_cdclk;
2758
2759         if (!crtc_state->hw.enable)
2760                 return 0;
2761
2762         min_cdclk = intel_pixel_rate_to_cdclk(crtc_state);
2763
2764         /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
2765         if (IS_BROADWELL(dev_priv) && hsw_crtc_state_ips_capable(crtc_state))
2766                 min_cdclk = DIV_ROUND_UP(min_cdclk * 100, 95);
2767
2768         /* BSpec says "Do not use DisplayPort with CDCLK less than 432 MHz,
2769          * audio enabled, port width x4, and link rate HBR2 (5.4 GHz), or else
2770          * there may be audio corruption or screen corruption." This cdclk
2771          * restriction for GLK is 316.8 MHz.
2772          */
2773         if (intel_crtc_has_dp_encoder(crtc_state) &&
2774             crtc_state->has_audio &&
2775             crtc_state->port_clock >= 540000 &&
2776             crtc_state->lane_count == 4) {
2777                 if (DISPLAY_VER(dev_priv) == 10) {
2778                         /* Display WA #1145: glk */
2779                         min_cdclk = max(316800, min_cdclk);
2780                 } else if (DISPLAY_VER(dev_priv) == 9 || IS_BROADWELL(dev_priv)) {
2781                         /* Display WA #1144: skl,bxt */
2782                         min_cdclk = max(432000, min_cdclk);
2783                 }
2784         }
2785
2786         /*
2787          * According to BSpec, "The CD clock frequency must be at least twice
2788          * the frequency of the Azalia BCLK." and BCLK is 96 MHz by default.
2789          */
2790         if (crtc_state->has_audio && DISPLAY_VER(dev_priv) >= 9)
2791                 min_cdclk = max(2 * 96000, min_cdclk);
2792
2793         /*
2794          * "For DP audio configuration, cdclk frequency shall be set to
2795          *  meet the following requirements:
2796          *  DP Link Frequency(MHz) | Cdclk frequency(MHz)
2797          *  270                    | 320 or higher
2798          *  162                    | 200 or higher"
2799          */
2800         if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
2801             intel_crtc_has_dp_encoder(crtc_state) && crtc_state->has_audio)
2802                 min_cdclk = max(crtc_state->port_clock, min_cdclk);
2803
2804         /*
2805          * On Valleyview some DSI panels lose (v|h)sync when the clock is lower
2806          * than 320000KHz.
2807          */
2808         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2809             IS_VALLEYVIEW(dev_priv))
2810                 min_cdclk = max(320000, min_cdclk);
2811
2812         /*
2813          * On Geminilake once the CDCLK gets as low as 79200
2814          * picture gets unstable, despite that values are
2815          * correct for DSI PLL and DE PLL.
2816          */
2817         if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI) &&
2818             IS_GEMINILAKE(dev_priv))
2819                 min_cdclk = max(158400, min_cdclk);
2820
2821         /* Account for additional needs from the planes */
2822         min_cdclk = max(intel_planes_min_cdclk(crtc_state), min_cdclk);
2823
2824         if (crtc_state->dsc.compression_enable)
2825                 min_cdclk = max(min_cdclk, intel_vdsc_min_cdclk(crtc_state));
2826
2827         return min_cdclk;
2828 }
2829
2830 static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
2831 {
2832         struct intel_atomic_state *state = cdclk_state->base.state;
2833         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2834         const struct intel_bw_state *bw_state;
2835         struct intel_crtc *crtc;
2836         struct intel_crtc_state *crtc_state;
2837         int min_cdclk, i;
2838         enum pipe pipe;
2839
2840         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2841                 int ret;
2842
2843                 min_cdclk = intel_crtc_compute_min_cdclk(crtc_state);
2844                 if (min_cdclk < 0)
2845                         return min_cdclk;
2846
2847                 if (cdclk_state->min_cdclk[crtc->pipe] == min_cdclk)
2848                         continue;
2849
2850                 cdclk_state->min_cdclk[crtc->pipe] = min_cdclk;
2851
2852                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2853                 if (ret)
2854                         return ret;
2855         }
2856
2857         bw_state = intel_atomic_get_new_bw_state(state);
2858         if (bw_state) {
2859                 min_cdclk = intel_bw_min_cdclk(dev_priv, bw_state);
2860
2861                 if (cdclk_state->bw_min_cdclk != min_cdclk) {
2862                         int ret;
2863
2864                         cdclk_state->bw_min_cdclk = min_cdclk;
2865
2866                         ret = intel_atomic_lock_global_state(&cdclk_state->base);
2867                         if (ret)
2868                                 return ret;
2869                 }
2870         }
2871
2872         min_cdclk = max(cdclk_state->force_min_cdclk,
2873                         cdclk_state->bw_min_cdclk);
2874         for_each_pipe(dev_priv, pipe)
2875                 min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
2876
2877         /*
2878          * Avoid glk_force_audio_cdclk() causing excessive screen
2879          * blinking when multiple pipes are active by making sure
2880          * CDCLK frequency is always high enough for audio. With a
2881          * single active pipe we can always change CDCLK frequency
2882          * by changing the cd2x divider (see glk_cdclk_table[]) and
2883          * thus a full modeset won't be needed then.
2884          */
2885         if (IS_GEMINILAKE(dev_priv) && cdclk_state->active_pipes &&
2886             !is_power_of_2(cdclk_state->active_pipes))
2887                 min_cdclk = max(2 * 96000, min_cdclk);
2888
2889         if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) {
2890                 drm_dbg_kms(&dev_priv->drm,
2891                             "required cdclk (%d kHz) exceeds max (%d kHz)\n",
2892                             min_cdclk, dev_priv->display.cdclk.max_cdclk_freq);
2893                 return -EINVAL;
2894         }
2895
2896         return min_cdclk;
2897 }
2898
2899 /*
2900  * Account for port clock min voltage level requirements.
2901  * This only really does something on DISPLA_VER >= 11 but can be
2902  * called on earlier platforms as well.
2903  *
2904  * Note that this functions assumes that 0 is
2905  * the lowest voltage value, and higher values
2906  * correspond to increasingly higher voltages.
2907  *
2908  * Should that relationship no longer hold on
2909  * future platforms this code will need to be
2910  * adjusted.
2911  */
2912 static int bxt_compute_min_voltage_level(struct intel_cdclk_state *cdclk_state)
2913 {
2914         struct intel_atomic_state *state = cdclk_state->base.state;
2915         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2916         struct intel_crtc *crtc;
2917         struct intel_crtc_state *crtc_state;
2918         u8 min_voltage_level;
2919         int i;
2920         enum pipe pipe;
2921
2922         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
2923                 int ret;
2924
2925                 if (crtc_state->hw.enable)
2926                         min_voltage_level = crtc_state->min_voltage_level;
2927                 else
2928                         min_voltage_level = 0;
2929
2930                 if (cdclk_state->min_voltage_level[crtc->pipe] == min_voltage_level)
2931                         continue;
2932
2933                 cdclk_state->min_voltage_level[crtc->pipe] = min_voltage_level;
2934
2935                 ret = intel_atomic_lock_global_state(&cdclk_state->base);
2936                 if (ret)
2937                         return ret;
2938         }
2939
2940         min_voltage_level = 0;
2941         for_each_pipe(dev_priv, pipe)
2942                 min_voltage_level = max(cdclk_state->min_voltage_level[pipe],
2943                                         min_voltage_level);
2944
2945         return min_voltage_level;
2946 }
2947
2948 static int vlv_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2949 {
2950         struct intel_atomic_state *state = cdclk_state->base.state;
2951         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2952         int min_cdclk, cdclk;
2953
2954         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2955         if (min_cdclk < 0)
2956                 return min_cdclk;
2957
2958         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
2959
2960         cdclk_state->logical.cdclk = cdclk;
2961         cdclk_state->logical.voltage_level =
2962                 vlv_calc_voltage_level(dev_priv, cdclk);
2963
2964         if (!cdclk_state->active_pipes) {
2965                 cdclk = vlv_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
2966
2967                 cdclk_state->actual.cdclk = cdclk;
2968                 cdclk_state->actual.voltage_level =
2969                         vlv_calc_voltage_level(dev_priv, cdclk);
2970         } else {
2971                 cdclk_state->actual = cdclk_state->logical;
2972         }
2973
2974         return 0;
2975 }
2976
2977 static int bdw_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
2978 {
2979         int min_cdclk, cdclk;
2980
2981         min_cdclk = intel_compute_min_cdclk(cdclk_state);
2982         if (min_cdclk < 0)
2983                 return min_cdclk;
2984
2985         cdclk = bdw_calc_cdclk(min_cdclk);
2986
2987         cdclk_state->logical.cdclk = cdclk;
2988         cdclk_state->logical.voltage_level =
2989                 bdw_calc_voltage_level(cdclk);
2990
2991         if (!cdclk_state->active_pipes) {
2992                 cdclk = bdw_calc_cdclk(cdclk_state->force_min_cdclk);
2993
2994                 cdclk_state->actual.cdclk = cdclk;
2995                 cdclk_state->actual.voltage_level =
2996                         bdw_calc_voltage_level(cdclk);
2997         } else {
2998                 cdclk_state->actual = cdclk_state->logical;
2999         }
3000
3001         return 0;
3002 }
3003
3004 static int skl_dpll0_vco(struct intel_cdclk_state *cdclk_state)
3005 {
3006         struct intel_atomic_state *state = cdclk_state->base.state;
3007         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3008         struct intel_crtc *crtc;
3009         struct intel_crtc_state *crtc_state;
3010         int vco, i;
3011
3012         vco = cdclk_state->logical.vco;
3013         if (!vco)
3014                 vco = dev_priv->skl_preferred_vco_freq;
3015
3016         for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
3017                 if (!crtc_state->hw.enable)
3018                         continue;
3019
3020                 if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
3021                         continue;
3022
3023                 /*
3024                  * DPLL0 VCO may need to be adjusted to get the correct
3025                  * clock for eDP. This will affect cdclk as well.
3026                  */
3027                 switch (crtc_state->port_clock / 2) {
3028                 case 108000:
3029                 case 216000:
3030                         vco = 8640000;
3031                         break;
3032                 default:
3033                         vco = 8100000;
3034                         break;
3035                 }
3036         }
3037
3038         return vco;
3039 }
3040
3041 static int skl_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
3042 {
3043         int min_cdclk, cdclk, vco;
3044
3045         min_cdclk = intel_compute_min_cdclk(cdclk_state);
3046         if (min_cdclk < 0)
3047                 return min_cdclk;
3048
3049         vco = skl_dpll0_vco(cdclk_state);
3050
3051         cdclk = skl_calc_cdclk(min_cdclk, vco);
3052
3053         cdclk_state->logical.vco = vco;
3054         cdclk_state->logical.cdclk = cdclk;
3055         cdclk_state->logical.voltage_level =
3056                 skl_calc_voltage_level(cdclk);
3057
3058         if (!cdclk_state->active_pipes) {
3059                 cdclk = skl_calc_cdclk(cdclk_state->force_min_cdclk, vco);
3060
3061                 cdclk_state->actual.vco = vco;
3062                 cdclk_state->actual.cdclk = cdclk;
3063                 cdclk_state->actual.voltage_level =
3064                         skl_calc_voltage_level(cdclk);
3065         } else {
3066                 cdclk_state->actual = cdclk_state->logical;
3067         }
3068
3069         return 0;
3070 }
3071
3072 static int bxt_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
3073 {
3074         struct intel_atomic_state *state = cdclk_state->base.state;
3075         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3076         int min_cdclk, min_voltage_level, cdclk, vco;
3077
3078         min_cdclk = intel_compute_min_cdclk(cdclk_state);
3079         if (min_cdclk < 0)
3080                 return min_cdclk;
3081
3082         min_voltage_level = bxt_compute_min_voltage_level(cdclk_state);
3083         if (min_voltage_level < 0)
3084                 return min_voltage_level;
3085
3086         cdclk = bxt_calc_cdclk(dev_priv, min_cdclk);
3087         vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
3088
3089         cdclk_state->logical.vco = vco;
3090         cdclk_state->logical.cdclk = cdclk;
3091         cdclk_state->logical.voltage_level =
3092                 max_t(int, min_voltage_level,
3093                       intel_cdclk_calc_voltage_level(dev_priv, cdclk));
3094
3095         if (!cdclk_state->active_pipes) {
3096                 cdclk = bxt_calc_cdclk(dev_priv, cdclk_state->force_min_cdclk);
3097                 vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
3098
3099                 cdclk_state->actual.vco = vco;
3100                 cdclk_state->actual.cdclk = cdclk;
3101                 cdclk_state->actual.voltage_level =
3102                         intel_cdclk_calc_voltage_level(dev_priv, cdclk);
3103         } else {
3104                 cdclk_state->actual = cdclk_state->logical;
3105         }
3106
3107         return 0;
3108 }
3109
3110 static int fixed_modeset_calc_cdclk(struct intel_cdclk_state *cdclk_state)
3111 {
3112         int min_cdclk;
3113
3114         /*
3115          * We can't change the cdclk frequency, but we still want to
3116          * check that the required minimum frequency doesn't exceed
3117          * the actual cdclk frequency.
3118          */
3119         min_cdclk = intel_compute_min_cdclk(cdclk_state);
3120         if (min_cdclk < 0)
3121                 return min_cdclk;
3122
3123         return 0;
3124 }
3125
3126 static struct intel_global_state *intel_cdclk_duplicate_state(struct intel_global_obj *obj)
3127 {
3128         struct intel_cdclk_state *cdclk_state;
3129
3130         cdclk_state = kmemdup(obj->state, sizeof(*cdclk_state), GFP_KERNEL);
3131         if (!cdclk_state)
3132                 return NULL;
3133
3134         cdclk_state->pipe = INVALID_PIPE;
3135         cdclk_state->disable_pipes = false;
3136
3137         return &cdclk_state->base;
3138 }
3139
3140 static void intel_cdclk_destroy_state(struct intel_global_obj *obj,
3141                                       struct intel_global_state *state)
3142 {
3143         kfree(state);
3144 }
3145
3146 static const struct intel_global_state_funcs intel_cdclk_funcs = {
3147         .atomic_duplicate_state = intel_cdclk_duplicate_state,
3148         .atomic_destroy_state = intel_cdclk_destroy_state,
3149 };
3150
3151 struct intel_cdclk_state *
3152 intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
3153 {
3154         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3155         struct intel_global_state *cdclk_state;
3156
3157         cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj);
3158         if (IS_ERR(cdclk_state))
3159                 return ERR_CAST(cdclk_state);
3160
3161         return to_intel_cdclk_state(cdclk_state);
3162 }
3163
3164 int intel_cdclk_atomic_check(struct intel_atomic_state *state,
3165                              bool *need_cdclk_calc)
3166 {
3167         const struct intel_cdclk_state *old_cdclk_state;
3168         const struct intel_cdclk_state *new_cdclk_state;
3169         struct intel_plane_state __maybe_unused *plane_state;
3170         struct intel_plane *plane;
3171         int ret;
3172         int i;
3173
3174         /*
3175          * active_planes bitmask has been updated, and potentially affected
3176          * planes are part of the state. We can now compute the minimum cdclk
3177          * for each plane.
3178          */
3179         for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
3180                 ret = intel_plane_calc_min_cdclk(state, plane, need_cdclk_calc);
3181                 if (ret)
3182                         return ret;
3183         }
3184
3185         ret = intel_bw_calc_min_cdclk(state, need_cdclk_calc);
3186         if (ret)
3187                 return ret;
3188
3189         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3190         new_cdclk_state = intel_atomic_get_new_cdclk_state(state);
3191
3192         if (new_cdclk_state &&
3193             old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk)
3194                 *need_cdclk_calc = true;
3195
3196         return 0;
3197 }
3198
3199 int intel_cdclk_state_set_joined_mbus(struct intel_atomic_state *state, bool joined_mbus)
3200 {
3201         struct intel_cdclk_state *cdclk_state;
3202
3203         cdclk_state = intel_atomic_get_cdclk_state(state);
3204         if (IS_ERR(cdclk_state))
3205                 return PTR_ERR(cdclk_state);
3206
3207         cdclk_state->actual.joined_mbus = joined_mbus;
3208         cdclk_state->logical.joined_mbus = joined_mbus;
3209
3210         return intel_atomic_lock_global_state(&cdclk_state->base);
3211 }
3212
3213 int intel_cdclk_init(struct drm_i915_private *dev_priv)
3214 {
3215         struct intel_cdclk_state *cdclk_state;
3216
3217         cdclk_state = kzalloc(sizeof(*cdclk_state), GFP_KERNEL);
3218         if (!cdclk_state)
3219                 return -ENOMEM;
3220
3221         intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj,
3222                                      &cdclk_state->base, &intel_cdclk_funcs);
3223
3224         return 0;
3225 }
3226
3227 static bool intel_cdclk_need_serialize(struct drm_i915_private *i915,
3228                                        const struct intel_cdclk_state *old_cdclk_state,
3229                                        const struct intel_cdclk_state *new_cdclk_state)
3230 {
3231         bool power_well_cnt_changed = hweight8(old_cdclk_state->active_pipes) !=
3232                                       hweight8(new_cdclk_state->active_pipes);
3233         bool cdclk_changed = intel_cdclk_changed(&old_cdclk_state->actual,
3234                                                  &new_cdclk_state->actual);
3235         /*
3236          * We need to poke hw for gen >= 12, because we notify PCode if
3237          * pipe power well count changes.
3238          */
3239         return cdclk_changed || (IS_DG2(i915) && power_well_cnt_changed);
3240 }
3241
3242 int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
3243 {
3244         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3245         const struct intel_cdclk_state *old_cdclk_state;
3246         struct intel_cdclk_state *new_cdclk_state;
3247         enum pipe pipe = INVALID_PIPE;
3248         int ret;
3249
3250         new_cdclk_state = intel_atomic_get_cdclk_state(state);
3251         if (IS_ERR(new_cdclk_state))
3252                 return PTR_ERR(new_cdclk_state);
3253
3254         old_cdclk_state = intel_atomic_get_old_cdclk_state(state);
3255
3256         new_cdclk_state->active_pipes =
3257                 intel_calc_active_pipes(state, old_cdclk_state->active_pipes);
3258
3259         ret = intel_cdclk_modeset_calc_cdclk(dev_priv, new_cdclk_state);
3260         if (ret)
3261                 return ret;
3262
3263         if (intel_cdclk_need_serialize(dev_priv, old_cdclk_state, new_cdclk_state)) {
3264                 /*
3265                  * Also serialize commits across all crtcs
3266                  * if the actual hw needs to be poked.
3267                  */
3268                 ret = intel_atomic_serialize_global_state(&new_cdclk_state->base);
3269                 if (ret)
3270                         return ret;
3271         } else if (old_cdclk_state->active_pipes != new_cdclk_state->active_pipes ||
3272                    old_cdclk_state->force_min_cdclk != new_cdclk_state->force_min_cdclk ||
3273                    intel_cdclk_changed(&old_cdclk_state->logical,
3274                                        &new_cdclk_state->logical)) {
3275                 ret = intel_atomic_lock_global_state(&new_cdclk_state->base);
3276                 if (ret)
3277                         return ret;
3278         } else {
3279                 return 0;
3280         }
3281
3282         if (is_power_of_2(new_cdclk_state->active_pipes) &&
3283             intel_cdclk_can_cd2x_update(dev_priv,
3284                                         &old_cdclk_state->actual,
3285                                         &new_cdclk_state->actual)) {
3286                 struct intel_crtc *crtc;
3287                 struct intel_crtc_state *crtc_state;
3288
3289                 pipe = ilog2(new_cdclk_state->active_pipes);
3290                 crtc = intel_crtc_for_pipe(dev_priv, pipe);
3291
3292                 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
3293                 if (IS_ERR(crtc_state))
3294                         return PTR_ERR(crtc_state);
3295
3296                 if (intel_crtc_needs_modeset(crtc_state))
3297                         pipe = INVALID_PIPE;
3298         }
3299
3300         if (intel_cdclk_can_crawl_and_squash(dev_priv,
3301                                              &old_cdclk_state->actual,
3302                                              &new_cdclk_state->actual)) {
3303                 drm_dbg_kms(&dev_priv->drm,
3304                             "Can change cdclk via crawling and squashing\n");
3305         } else if (intel_cdclk_can_squash(dev_priv,
3306                                         &old_cdclk_state->actual,
3307                                         &new_cdclk_state->actual)) {
3308                 drm_dbg_kms(&dev_priv->drm,
3309                             "Can change cdclk via squashing\n");
3310         } else if (intel_cdclk_can_crawl(dev_priv,
3311                                          &old_cdclk_state->actual,
3312                                          &new_cdclk_state->actual)) {
3313                 drm_dbg_kms(&dev_priv->drm,
3314                             "Can change cdclk via crawling\n");
3315         } else if (pipe != INVALID_PIPE) {
3316                 new_cdclk_state->pipe = pipe;
3317
3318                 drm_dbg_kms(&dev_priv->drm,
3319                             "Can change cdclk cd2x divider with pipe %c active\n",
3320                             pipe_name(pipe));
3321         } else if (intel_cdclk_clock_changed(&old_cdclk_state->actual,
3322                                              &new_cdclk_state->actual)) {
3323                 /* All pipes must be switched off while we change the cdclk. */
3324                 ret = intel_modeset_all_pipes_late(state, "CDCLK change");
3325                 if (ret)
3326                         return ret;
3327
3328                 new_cdclk_state->disable_pipes = true;
3329
3330                 drm_dbg_kms(&dev_priv->drm,
3331                             "Modeset required for cdclk change\n");
3332         }
3333
3334         if (intel_mdclk_cdclk_ratio(dev_priv, &old_cdclk_state->actual) !=
3335             intel_mdclk_cdclk_ratio(dev_priv, &new_cdclk_state->actual)) {
3336                 int ratio = intel_mdclk_cdclk_ratio(dev_priv, &new_cdclk_state->actual);
3337
3338                 ret = intel_dbuf_state_set_mdclk_cdclk_ratio(state, ratio);
3339                 if (ret)
3340                         return ret;
3341         }
3342
3343         drm_dbg_kms(&dev_priv->drm,
3344                     "New cdclk calculated to be logical %u kHz, actual %u kHz\n",
3345                     new_cdclk_state->logical.cdclk,
3346                     new_cdclk_state->actual.cdclk);
3347         drm_dbg_kms(&dev_priv->drm,
3348                     "New voltage level calculated to be logical %u, actual %u\n",
3349                     new_cdclk_state->logical.voltage_level,
3350                     new_cdclk_state->actual.voltage_level);
3351
3352         return 0;
3353 }
3354
3355 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
3356 {
3357         int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq;
3358
3359         if (DISPLAY_VER(dev_priv) >= 10)
3360                 return 2 * max_cdclk_freq;
3361         else if (DISPLAY_VER(dev_priv) == 9 ||
3362                  IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3363                 return max_cdclk_freq;
3364         else if (IS_CHERRYVIEW(dev_priv))
3365                 return max_cdclk_freq*95/100;
3366         else if (DISPLAY_VER(dev_priv) < 4)
3367                 return 2*max_cdclk_freq*90/100;
3368         else
3369                 return max_cdclk_freq*90/100;
3370 }
3371
3372 /**
3373  * intel_update_max_cdclk - Determine the maximum support CDCLK frequency
3374  * @dev_priv: i915 device
3375  *
3376  * Determine the maximum CDCLK frequency the platform supports, and also
3377  * derive the maximum dot clock frequency the maximum CDCLK frequency
3378  * allows.
3379  */
3380 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
3381 {
3382         if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3383                 if (dev_priv->display.cdclk.hw.ref == 24000)
3384                         dev_priv->display.cdclk.max_cdclk_freq = 552000;
3385                 else
3386                         dev_priv->display.cdclk.max_cdclk_freq = 556800;
3387         } else if (DISPLAY_VER(dev_priv) >= 11) {
3388                 if (dev_priv->display.cdclk.hw.ref == 24000)
3389                         dev_priv->display.cdclk.max_cdclk_freq = 648000;
3390                 else
3391                         dev_priv->display.cdclk.max_cdclk_freq = 652800;
3392         } else if (IS_GEMINILAKE(dev_priv)) {
3393                 dev_priv->display.cdclk.max_cdclk_freq = 316800;
3394         } else if (IS_BROXTON(dev_priv)) {
3395                 dev_priv->display.cdclk.max_cdclk_freq = 624000;
3396         } else if (DISPLAY_VER(dev_priv) == 9) {
3397                 u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
3398                 int max_cdclk, vco;
3399
3400                 vco = dev_priv->skl_preferred_vco_freq;
3401                 drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000);
3402
3403                 /*
3404                  * Use the lower (vco 8640) cdclk values as a
3405                  * first guess. skl_calc_cdclk() will correct it
3406                  * if the preferred vco is 8100 instead.
3407                  */
3408                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
3409                         max_cdclk = 617143;
3410                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
3411                         max_cdclk = 540000;
3412                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
3413                         max_cdclk = 432000;
3414                 else
3415                         max_cdclk = 308571;
3416
3417                 dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
3418         } else if (IS_BROADWELL(dev_priv))  {
3419                 /*
3420                  * FIXME with extra cooling we can allow
3421                  * 540 MHz for ULX and 675 Mhz for ULT.
3422                  * How can we know if extra cooling is
3423                  * available? PCI ID, VTB, something else?
3424                  */
3425                 if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
3426                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
3427                 else if (IS_BROADWELL_ULX(dev_priv))
3428                         dev_priv->display.cdclk.max_cdclk_freq = 450000;
3429                 else if (IS_BROADWELL_ULT(dev_priv))
3430                         dev_priv->display.cdclk.max_cdclk_freq = 540000;
3431                 else
3432                         dev_priv->display.cdclk.max_cdclk_freq = 675000;
3433         } else if (IS_CHERRYVIEW(dev_priv)) {
3434                 dev_priv->display.cdclk.max_cdclk_freq = 320000;
3435         } else if (IS_VALLEYVIEW(dev_priv)) {
3436                 dev_priv->display.cdclk.max_cdclk_freq = 400000;
3437         } else {
3438                 /* otherwise assume cdclk is fixed */
3439                 dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
3440         }
3441
3442         dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
3443
3444         drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
3445                 dev_priv->display.cdclk.max_cdclk_freq);
3446
3447         drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
3448                 dev_priv->max_dotclk_freq);
3449 }
3450
3451 /**
3452  * intel_update_cdclk - Determine the current CDCLK frequency
3453  * @dev_priv: i915 device
3454  *
3455  * Determine the current CDCLK frequency.
3456  */
3457 void intel_update_cdclk(struct drm_i915_private *dev_priv)
3458 {
3459         intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw);
3460
3461         /*
3462          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
3463          * Programmng [sic] note: bit[9:2] should be programmed to the number
3464          * of cdclk that generates 4MHz reference clock freq which is used to
3465          * generate GMBus clock. This will vary with the cdclk freq.
3466          */
3467         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3468                 intel_de_write(dev_priv, GMBUSFREQ_VLV,
3469                                DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000));
3470 }
3471
3472 static int dg1_rawclk(struct drm_i915_private *dev_priv)
3473 {
3474         /*
3475          * DG1 always uses a 38.4 MHz rawclk.  The bspec tells us
3476          * "Program Numerator=2, Denominator=4, Divider=37 decimal."
3477          */
3478         intel_de_write(dev_priv, PCH_RAWCLK_FREQ,
3479                        CNP_RAWCLK_DEN(4) | CNP_RAWCLK_DIV(37) | ICP_RAWCLK_NUM(2));
3480
3481         return 38400;
3482 }
3483
3484 static int cnp_rawclk(struct drm_i915_private *dev_priv)
3485 {
3486         u32 rawclk;
3487         int divider, fraction;
3488
3489         if (intel_de_read(dev_priv, SFUSE_STRAP) & SFUSE_STRAP_RAW_FREQUENCY) {
3490                 /* 24 MHz */
3491                 divider = 24000;
3492                 fraction = 0;
3493         } else {
3494                 /* 19.2 MHz */
3495                 divider = 19000;
3496                 fraction = 200;
3497         }
3498
3499         rawclk = CNP_RAWCLK_DIV(divider / 1000);
3500         if (fraction) {
3501                 int numerator = 1;
3502
3503                 rawclk |= CNP_RAWCLK_DEN(DIV_ROUND_CLOSEST(numerator * 1000,
3504                                                            fraction) - 1);
3505                 if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
3506                         rawclk |= ICP_RAWCLK_NUM(numerator);
3507         }
3508
3509         intel_de_write(dev_priv, PCH_RAWCLK_FREQ, rawclk);
3510         return divider + fraction;
3511 }
3512
3513 static int pch_rawclk(struct drm_i915_private *dev_priv)
3514 {
3515         return (intel_de_read(dev_priv, PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK) * 1000;
3516 }
3517
3518 static int vlv_hrawclk(struct drm_i915_private *dev_priv)
3519 {
3520         /* RAWCLK_FREQ_VLV register updated from power well code */
3521         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
3522                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
3523 }
3524
3525 static int i9xx_hrawclk(struct drm_i915_private *dev_priv)
3526 {
3527         u32 clkcfg;
3528
3529         /*
3530          * hrawclock is 1/4 the FSB frequency
3531          *
3532          * Note that this only reads the state of the FSB
3533          * straps, not the actual FSB frequency. Some BIOSen
3534          * let you configure each independently. Ideally we'd
3535          * read out the actual FSB frequency but sadly we
3536          * don't know which registers have that information,
3537          * and all the relevant docs have gone to bit heaven :(
3538          */
3539         clkcfg = intel_de_read(dev_priv, CLKCFG) & CLKCFG_FSB_MASK;
3540
3541         if (IS_MOBILE(dev_priv)) {
3542                 switch (clkcfg) {
3543                 case CLKCFG_FSB_400:
3544                         return 100000;
3545                 case CLKCFG_FSB_533:
3546                         return 133333;
3547                 case CLKCFG_FSB_667:
3548                         return 166667;
3549                 case CLKCFG_FSB_800:
3550                         return 200000;
3551                 case CLKCFG_FSB_1067:
3552                         return 266667;
3553                 case CLKCFG_FSB_1333:
3554                         return 333333;
3555                 default:
3556                         MISSING_CASE(clkcfg);
3557                         return 133333;
3558                 }
3559         } else {
3560                 switch (clkcfg) {
3561                 case CLKCFG_FSB_400_ALT:
3562                         return 100000;
3563                 case CLKCFG_FSB_533:
3564                         return 133333;
3565                 case CLKCFG_FSB_667:
3566                         return 166667;
3567                 case CLKCFG_FSB_800:
3568                         return 200000;
3569                 case CLKCFG_FSB_1067_ALT:
3570                         return 266667;
3571                 case CLKCFG_FSB_1333_ALT:
3572                         return 333333;
3573                 case CLKCFG_FSB_1600_ALT:
3574                         return 400000;
3575                 default:
3576                         return 133333;
3577                 }
3578         }
3579 }
3580
3581 /**
3582  * intel_read_rawclk - Determine the current RAWCLK frequency
3583  * @dev_priv: i915 device
3584  *
3585  * Determine the current RAWCLK frequency. RAWCLK is a fixed
3586  * frequency clock so this needs to done only once.
3587  */
3588 u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
3589 {
3590         u32 freq;
3591
3592         if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTL)
3593                 /*
3594                  * MTL always uses a 38.4 MHz rawclk.  The bspec tells us
3595                  * "RAWCLK_FREQ defaults to the values for 38.4 and does
3596                  * not need to be programmed."
3597                  */
3598                 freq = 38400;
3599         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
3600                 freq = dg1_rawclk(dev_priv);
3601         else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
3602                 freq = cnp_rawclk(dev_priv);
3603         else if (HAS_PCH_SPLIT(dev_priv))
3604                 freq = pch_rawclk(dev_priv);
3605         else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
3606                 freq = vlv_hrawclk(dev_priv);
3607         else if (DISPLAY_VER(dev_priv) >= 3)
3608                 freq = i9xx_hrawclk(dev_priv);
3609         else
3610                 /* no rawclk on other platforms, or no need to know it */
3611                 return 0;
3612
3613         return freq;
3614 }
3615
3616 static int i915_cdclk_info_show(struct seq_file *m, void *unused)
3617 {
3618         struct drm_i915_private *i915 = m->private;
3619
3620         seq_printf(m, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk);
3621         seq_printf(m, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq);
3622         seq_printf(m, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq);
3623
3624         return 0;
3625 }
3626
3627 DEFINE_SHOW_ATTRIBUTE(i915_cdclk_info);
3628
3629 void intel_cdclk_debugfs_register(struct drm_i915_private *i915)
3630 {
3631         struct drm_minor *minor = i915->drm.primary;
3632
3633         debugfs_create_file("i915_cdclk_info", 0444, minor->debugfs_root,
3634                             i915, &i915_cdclk_info_fops);
3635 }
3636
3637 static const struct intel_cdclk_funcs rplu_cdclk_funcs = {
3638         .get_cdclk = bxt_get_cdclk,
3639         .set_cdclk = bxt_set_cdclk,
3640         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3641         .calc_voltage_level = rplu_calc_voltage_level,
3642 };
3643
3644 static const struct intel_cdclk_funcs tgl_cdclk_funcs = {
3645         .get_cdclk = bxt_get_cdclk,
3646         .set_cdclk = bxt_set_cdclk,
3647         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3648         .calc_voltage_level = tgl_calc_voltage_level,
3649 };
3650
3651 static const struct intel_cdclk_funcs ehl_cdclk_funcs = {
3652         .get_cdclk = bxt_get_cdclk,
3653         .set_cdclk = bxt_set_cdclk,
3654         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3655         .calc_voltage_level = ehl_calc_voltage_level,
3656 };
3657
3658 static const struct intel_cdclk_funcs icl_cdclk_funcs = {
3659         .get_cdclk = bxt_get_cdclk,
3660         .set_cdclk = bxt_set_cdclk,
3661         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3662         .calc_voltage_level = icl_calc_voltage_level,
3663 };
3664
3665 static const struct intel_cdclk_funcs bxt_cdclk_funcs = {
3666         .get_cdclk = bxt_get_cdclk,
3667         .set_cdclk = bxt_set_cdclk,
3668         .modeset_calc_cdclk = bxt_modeset_calc_cdclk,
3669         .calc_voltage_level = bxt_calc_voltage_level,
3670 };
3671
3672 static const struct intel_cdclk_funcs skl_cdclk_funcs = {
3673         .get_cdclk = skl_get_cdclk,
3674         .set_cdclk = skl_set_cdclk,
3675         .modeset_calc_cdclk = skl_modeset_calc_cdclk,
3676 };
3677
3678 static const struct intel_cdclk_funcs bdw_cdclk_funcs = {
3679         .get_cdclk = bdw_get_cdclk,
3680         .set_cdclk = bdw_set_cdclk,
3681         .modeset_calc_cdclk = bdw_modeset_calc_cdclk,
3682 };
3683
3684 static const struct intel_cdclk_funcs chv_cdclk_funcs = {
3685         .get_cdclk = vlv_get_cdclk,
3686         .set_cdclk = chv_set_cdclk,
3687         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3688 };
3689
3690 static const struct intel_cdclk_funcs vlv_cdclk_funcs = {
3691         .get_cdclk = vlv_get_cdclk,
3692         .set_cdclk = vlv_set_cdclk,
3693         .modeset_calc_cdclk = vlv_modeset_calc_cdclk,
3694 };
3695
3696 static const struct intel_cdclk_funcs hsw_cdclk_funcs = {
3697         .get_cdclk = hsw_get_cdclk,
3698         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3699 };
3700
3701 /* SNB, IVB, 965G, 945G */
3702 static const struct intel_cdclk_funcs fixed_400mhz_cdclk_funcs = {
3703         .get_cdclk = fixed_400mhz_get_cdclk,
3704         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3705 };
3706
3707 static const struct intel_cdclk_funcs ilk_cdclk_funcs = {
3708         .get_cdclk = fixed_450mhz_get_cdclk,
3709         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3710 };
3711
3712 static const struct intel_cdclk_funcs gm45_cdclk_funcs = {
3713         .get_cdclk = gm45_get_cdclk,
3714         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3715 };
3716
3717 /* G45 uses G33 */
3718
3719 static const struct intel_cdclk_funcs i965gm_cdclk_funcs = {
3720         .get_cdclk = i965gm_get_cdclk,
3721         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3722 };
3723
3724 /* i965G uses fixed 400 */
3725
3726 static const struct intel_cdclk_funcs pnv_cdclk_funcs = {
3727         .get_cdclk = pnv_get_cdclk,
3728         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3729 };
3730
3731 static const struct intel_cdclk_funcs g33_cdclk_funcs = {
3732         .get_cdclk = g33_get_cdclk,
3733         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3734 };
3735
3736 static const struct intel_cdclk_funcs i945gm_cdclk_funcs = {
3737         .get_cdclk = i945gm_get_cdclk,
3738         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3739 };
3740
3741 /* i945G uses fixed 400 */
3742
3743 static const struct intel_cdclk_funcs i915gm_cdclk_funcs = {
3744         .get_cdclk = i915gm_get_cdclk,
3745         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3746 };
3747
3748 static const struct intel_cdclk_funcs i915g_cdclk_funcs = {
3749         .get_cdclk = fixed_333mhz_get_cdclk,
3750         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3751 };
3752
3753 static const struct intel_cdclk_funcs i865g_cdclk_funcs = {
3754         .get_cdclk = fixed_266mhz_get_cdclk,
3755         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3756 };
3757
3758 static const struct intel_cdclk_funcs i85x_cdclk_funcs = {
3759         .get_cdclk = i85x_get_cdclk,
3760         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3761 };
3762
3763 static const struct intel_cdclk_funcs i845g_cdclk_funcs = {
3764         .get_cdclk = fixed_200mhz_get_cdclk,
3765         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3766 };
3767
3768 static const struct intel_cdclk_funcs i830_cdclk_funcs = {
3769         .get_cdclk = fixed_133mhz_get_cdclk,
3770         .modeset_calc_cdclk = fixed_modeset_calc_cdclk,
3771 };
3772
3773 /**
3774  * intel_init_cdclk_hooks - Initialize CDCLK related modesetting hooks
3775  * @dev_priv: i915 device
3776  */
3777 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
3778 {
3779         if (DISPLAY_VER(dev_priv) >= 20) {
3780                 dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3781                 dev_priv->display.cdclk.table = xe2lpd_cdclk_table;
3782         } else if (DISPLAY_VER(dev_priv) >= 14) {
3783                 dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3784                 dev_priv->display.cdclk.table = mtl_cdclk_table;
3785         } else if (IS_DG2(dev_priv)) {
3786                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3787                 dev_priv->display.cdclk.table = dg2_cdclk_table;
3788         } else if (IS_ALDERLAKE_P(dev_priv)) {
3789                 /* Wa_22011320316:adl-p[a0] */
3790                 if (IS_ALDERLAKE_P(dev_priv) && IS_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0)) {
3791                         dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
3792                         dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3793                 } else if (IS_RAPTORLAKE_U(dev_priv)) {
3794                         dev_priv->display.cdclk.table = rplu_cdclk_table;
3795                         dev_priv->display.funcs.cdclk = &rplu_cdclk_funcs;
3796                 } else {
3797                         dev_priv->display.cdclk.table = adlp_cdclk_table;
3798                         dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3799                 }
3800         } else if (IS_ROCKETLAKE(dev_priv)) {
3801                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3802                 dev_priv->display.cdclk.table = rkl_cdclk_table;
3803         } else if (DISPLAY_VER(dev_priv) >= 12) {
3804                 dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
3805                 dev_priv->display.cdclk.table = icl_cdclk_table;
3806         } else if (IS_JASPERLAKE(dev_priv) || IS_ELKHARTLAKE(dev_priv)) {
3807                 dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs;
3808                 dev_priv->display.cdclk.table = icl_cdclk_table;
3809         } else if (DISPLAY_VER(dev_priv) >= 11) {
3810                 dev_priv->display.funcs.cdclk = &icl_cdclk_funcs;
3811                 dev_priv->display.cdclk.table = icl_cdclk_table;
3812         } else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
3813                 dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs;
3814                 if (IS_GEMINILAKE(dev_priv))
3815                         dev_priv->display.cdclk.table = glk_cdclk_table;
3816                 else
3817                         dev_priv->display.cdclk.table = bxt_cdclk_table;
3818         } else if (DISPLAY_VER(dev_priv) == 9) {
3819                 dev_priv->display.funcs.cdclk = &skl_cdclk_funcs;
3820         } else if (IS_BROADWELL(dev_priv)) {
3821                 dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs;
3822         } else if (IS_HASWELL(dev_priv)) {
3823                 dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs;
3824         } else if (IS_CHERRYVIEW(dev_priv)) {
3825                 dev_priv->display.funcs.cdclk = &chv_cdclk_funcs;
3826         } else if (IS_VALLEYVIEW(dev_priv)) {
3827                 dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs;
3828         } else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
3829                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3830         } else if (IS_IRONLAKE(dev_priv)) {
3831                 dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs;
3832         } else if (IS_GM45(dev_priv)) {
3833                 dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs;
3834         } else if (IS_G45(dev_priv)) {
3835                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3836         } else if (IS_I965GM(dev_priv)) {
3837                 dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs;
3838         } else if (IS_I965G(dev_priv)) {
3839                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3840         } else if (IS_PINEVIEW(dev_priv)) {
3841                 dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs;
3842         } else if (IS_G33(dev_priv)) {
3843                 dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
3844         } else if (IS_I945GM(dev_priv)) {
3845                 dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs;
3846         } else if (IS_I945G(dev_priv)) {
3847                 dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
3848         } else if (IS_I915GM(dev_priv)) {
3849                 dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs;
3850         } else if (IS_I915G(dev_priv)) {
3851                 dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs;
3852         } else if (IS_I865G(dev_priv)) {
3853                 dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs;
3854         } else if (IS_I85X(dev_priv)) {
3855                 dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs;
3856         } else if (IS_I845G(dev_priv)) {
3857                 dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs;
3858         } else if (IS_I830(dev_priv)) {
3859                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3860         }
3861
3862         if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk,
3863                      "Unknown platform. Assuming i830\n"))
3864                 dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
3865 }