drivers/devfreq/sun8i-a33-mbus.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 //
   3 // Copyright (C) 2020-2021 Samuel Holland <samuel@sholland.org>
   4 //
   5
   6 #include <linux/clk.h>
   7 #include <linux/devfreq.h>
   8 #include <linux/err.h>
   9 #include <linux/io.h>
  10 #include <linux/iopoll.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/platform_device.h>
  14 #include <linux/property.h>
  15
  16 #define MBUS_CR                         0x0000
  17 #define MBUS_CR_GET_DRAM_TYPE(x)        (((x) >> 16) & 0x7)
  18 #define MBUS_CR_DRAM_TYPE_DDR2          2
  19 #define MBUS_CR_DRAM_TYPE_DDR3          3
  20 #define MBUS_CR_DRAM_TYPE_DDR4          4
  21 #define MBUS_CR_DRAM_TYPE_LPDDR2        6
  22 #define MBUS_CR_DRAM_TYPE_LPDDR3        7
  23
  24 #define MBUS_TMR                        0x000c
  25 #define MBUS_TMR_PERIOD(x)              ((x) - 1)
  26
  27 #define MBUS_PMU_CFG                    0x009c
  28 #define MBUS_PMU_CFG_PERIOD(x)          (((x) - 1) << 16)
  29 #define MBUS_PMU_CFG_UNIT               (0x3 << 1)
  30 #define MBUS_PMU_CFG_UNIT_B             (0x0 << 1)
  31 #define MBUS_PMU_CFG_UNIT_KB            (0x1 << 1)
  32 #define MBUS_PMU_CFG_UNIT_MB            (0x2 << 1)
  33 #define MBUS_PMU_CFG_ENABLE             (0x1 << 0)
  34
  35 #define MBUS_PMU_BWCR(n)                (0x00a0 + (0x04 * (n)))
  36
  37 #define MBUS_TOTAL_BWCR                 MBUS_PMU_BWCR(5)
  38 #define MBUS_TOTAL_BWCR_H616            MBUS_PMU_BWCR(13)
  39
  40 #define MBUS_MDFSCR                     0x0100
  41 #define MBUS_MDFSCR_BUFFER_TIMING       (0x1 << 15)
  42 #define MBUS_MDFSCR_PAD_HOLD            (0x1 << 13)
  43 #define MBUS_MDFSCR_BYPASS              (0x1 << 4)
  44 #define MBUS_MDFSCR_MODE                (0x1 << 1)
  45 #define MBUS_MDFSCR_MODE_DFS            (0x0 << 1)
  46 #define MBUS_MDFSCR_MODE_CFS            (0x1 << 1)
  47 #define MBUS_MDFSCR_START               (0x1 << 0)
  48
  49 #define MBUS_MDFSMRMR                   0x0108
  50
  51 #define DRAM_PWRCTL                     0x0004
  52 #define DRAM_PWRCTL_SELFREF_EN          (0x1 << 0)
  53
  54 #define DRAM_RFSHTMG                    0x0090
  55 #define DRAM_RFSHTMG_TREFI(x)           ((x) << 16)
  56 #define DRAM_RFSHTMG_TRFC(x)            ((x) << 0)
  57
  58 #define DRAM_VTFCR                      0x00b8
  59 #define DRAM_VTFCR_VTF_ENABLE           (0x3 << 8)
  60
  61 #define DRAM_ODTMAP                     0x0120
  62
  63 #define DRAM_DX_MAX                     4
  64
  65 #define DRAM_DXnGCR0(n)                 (0x0344 + 0x80 * (n))
  66 #define DRAM_DXnGCR0_DXODT              (0x3 << 4)
  67 #define DRAM_DXnGCR0_DXODT_DYNAMIC      (0x0 << 4)
  68 #define DRAM_DXnGCR0_DXODT_ENABLED      (0x1 << 4)
  69 #define DRAM_DXnGCR0_DXODT_DISABLED     (0x2 << 4)
  70 #define DRAM_DXnGCR0_DXEN               (0x1 << 0)
  71
  72 struct sun8i_a33_mbus_variant {
  73         u32                                     min_dram_divider;
  74         u32                                     max_dram_divider;
  75         u32                                     odt_freq_mhz;
  76 };
  77
  78 struct sun8i_a33_mbus {
  79         const struct sun8i_a33_mbus_variant     *variant;
  80         void __iomem                            *reg_dram;
  81         void __iomem                            *reg_mbus;
  82         struct clk                              *clk_bus;
  83         struct clk                              *clk_dram;
  84         struct clk                              *clk_mbus;
  85         struct devfreq                          *devfreq_dram;
  86         struct devfreq_simple_ondemand_data     gov_data;
  87         struct devfreq_dev_profile              profile;
  88         u32                                     data_width;
  89         u32                                     nominal_bw;
  90         u32                                     odtmap;
  91         u32                                     tREFI_ns;
  92         u32                                     tRFC_ns;
  93         unsigned long                           freq_table[];
  94 };
  95
  96 /*
  97  * The unit for this value is (MBUS clock cycles / MBUS_TMR_PERIOD). When
  98  * MBUS_TMR_PERIOD is programmed to match the MBUS clock frequency in MHz, as
  99  * it is during DRAM init and during probe, the resulting unit is microseconds.
 100  */
 101 static int pmu_period = 50000;
 102 module_param(pmu_period, int, 0644);
 103 MODULE_PARM_DESC(pmu_period, "Bandwidth measurement period (microseconds)");
 104
 105 static u32 sun8i_a33_mbus_get_peak_bw(struct sun8i_a33_mbus *priv)
 106 {
 107         /* Returns the peak transfer (in KiB) during any single PMU period. */
 108         return readl_relaxed(priv->reg_mbus + MBUS_TOTAL_BWCR);
 109 }
 110
 111 static void sun8i_a33_mbus_restart_pmu_counters(struct sun8i_a33_mbus *priv)
 112 {
 113         u32 pmu_cfg = MBUS_PMU_CFG_PERIOD(pmu_period) | MBUS_PMU_CFG_UNIT_KB;
 114
 115         /* All PMU counters are cleared on a disable->enable transition. */
 116         writel_relaxed(pmu_cfg,
 117                        priv->reg_mbus + MBUS_PMU_CFG);
 118         writel_relaxed(pmu_cfg | MBUS_PMU_CFG_ENABLE,
 119                        priv->reg_mbus + MBUS_PMU_CFG);
 120
 121 }
 122
 123 static void sun8i_a33_mbus_update_nominal_bw(struct sun8i_a33_mbus *priv,
 124                                              u32 ddr_freq_mhz)
 125 {
 126         /*
 127          * Nominal bandwidth (KiB per PMU period):
 128          *
 129          *   DDR transfers   microseconds     KiB
 130          *   ------------- * ------------ * --------
 131          *    microsecond     PMU period    transfer
 132          */
 133         priv->nominal_bw = ddr_freq_mhz * pmu_period * priv->data_width / 1024;
 134 }
 135
 136 static int sun8i_a33_mbus_set_dram_freq(struct sun8i_a33_mbus *priv,
 137                                         unsigned long freq)
 138 {
 139         u32  ddr_freq_mhz = freq / USEC_PER_SEC; /* DDR */
 140         u32 dram_freq_mhz =    ddr_freq_mhz / 2; /* SDR */
 141         u32 mctl_freq_mhz =   dram_freq_mhz / 2; /* HDR */
 142         u32 dxodt, mdfscr, pwrctl, vtfcr;
 143         u32 i, tREFI_32ck, tRFC_ck;
 144         int ret;
 145
 146         /* The rate change is not effective until the MDFS process runs. */
 147         ret = clk_set_rate(priv->clk_dram, freq);
 148         if (ret)
 149                 return ret;
 150
 151         /* Disable automatic self-refesh and VTF before starting MDFS. */
 152         pwrctl = readl_relaxed(priv->reg_dram + DRAM_PWRCTL) &
 153                  ~DRAM_PWRCTL_SELFREF_EN;
 154         writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
 155         vtfcr = readl_relaxed(priv->reg_dram + DRAM_VTFCR);
 156         writel_relaxed(vtfcr & ~DRAM_VTFCR_VTF_ENABLE,
 157                        priv->reg_dram + DRAM_VTFCR);
 158
 159         /* Set up MDFS and enable double buffering for timing registers. */
 160         mdfscr = MBUS_MDFSCR_MODE_DFS |
 161                  MBUS_MDFSCR_BYPASS |
 162                  MBUS_MDFSCR_PAD_HOLD |
 163                  MBUS_MDFSCR_BUFFER_TIMING;
 164         writel(mdfscr, priv->reg_mbus + MBUS_MDFSCR);
 165
 166         /* Update the buffered copy of RFSHTMG. */
 167         tREFI_32ck = priv->tREFI_ns * mctl_freq_mhz / 1000 / 32;
 168         tRFC_ck = DIV_ROUND_UP(priv->tRFC_ns * mctl_freq_mhz, 1000);
 169         writel(DRAM_RFSHTMG_TREFI(tREFI_32ck) | DRAM_RFSHTMG_TRFC(tRFC_ck),
 170                priv->reg_dram + DRAM_RFSHTMG);
 171
 172         /* Enable ODT if needed, or disable it to save power. */
 173         if (priv->odtmap && dram_freq_mhz > priv->variant->odt_freq_mhz) {
 174                 dxodt = DRAM_DXnGCR0_DXODT_DYNAMIC;
 175                 writel(priv->odtmap, priv->reg_dram + DRAM_ODTMAP);
 176         } else {
 177                 dxodt = DRAM_DXnGCR0_DXODT_DISABLED;
 178                 writel(0, priv->reg_dram + DRAM_ODTMAP);
 179         }
 180         for (i = 0; i < DRAM_DX_MAX; ++i) {
 181                 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
 182
 183                 writel((readl(reg) & ~DRAM_DXnGCR0_DXODT) | dxodt, reg);
 184         }
 185
 186         dev_dbg(priv->devfreq_dram->dev.parent,
 187                 "Setting DRAM to %u MHz, tREFI=%u, tRFC=%u, ODT=%s\n",
 188                 dram_freq_mhz, tREFI_32ck, tRFC_ck,
 189                 dxodt == DRAM_DXnGCR0_DXODT_DYNAMIC ? "dynamic" : "disabled");
 190
 191         /* Trigger hardware MDFS. */
 192         writel(mdfscr | MBUS_MDFSCR_START, priv->reg_mbus + MBUS_MDFSCR);
 193         ret = readl_poll_timeout_atomic(priv->reg_mbus + MBUS_MDFSCR, mdfscr,
 194                                         !(mdfscr & MBUS_MDFSCR_START), 10, 1000);
 195         if (ret)
 196                 return ret;
 197
 198         /* Disable double buffering. */
 199         writel(0, priv->reg_mbus + MBUS_MDFSCR);
 200
 201         /* Restore VTF configuration. */
 202         writel_relaxed(vtfcr, priv->reg_dram + DRAM_VTFCR);
 203
 204         /* Enable automatic self-refresh at the lowest frequency only. */
 205         if (freq == priv->freq_table[0])
 206                 pwrctl |= DRAM_PWRCTL_SELFREF_EN;
 207         writel_relaxed(pwrctl, priv->reg_dram + DRAM_PWRCTL);
 208
 209         sun8i_a33_mbus_restart_pmu_counters(priv);
 210         sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq_mhz);
 211
 212         return 0;
 213 }
 214
 215 static int sun8i_a33_mbus_set_dram_target(struct device *dev,
 216                                           unsigned long *freq, u32 flags)
 217 {
 218         struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
 219         struct devfreq *devfreq = priv->devfreq_dram;
 220         struct dev_pm_opp *opp;
 221         int ret;
 222
 223         opp = devfreq_recommended_opp(dev, freq, flags);
 224         if (IS_ERR(opp))
 225                 return PTR_ERR(opp);
 226
 227         dev_pm_opp_put(opp);
 228
 229         if (*freq == devfreq->previous_freq)
 230                 return 0;
 231
 232         ret = sun8i_a33_mbus_set_dram_freq(priv, *freq);
 233         if (ret) {
 234                 dev_warn(dev, "failed to set DRAM frequency: %d\n", ret);
 235                 *freq = devfreq->previous_freq;
 236         }
 237
 238         return ret;
 239 }
 240
 241 static int sun8i_a33_mbus_get_dram_status(struct device *dev,
 242                                           struct devfreq_dev_status *stat)
 243 {
 244         struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
 245
 246         stat->busy_time         = sun8i_a33_mbus_get_peak_bw(priv);
 247         stat->total_time        = priv->nominal_bw;
 248         stat->current_frequency = priv->devfreq_dram->previous_freq;
 249
 250         sun8i_a33_mbus_restart_pmu_counters(priv);
 251
 252         dev_dbg(dev, "Using %lu/%lu (%lu%%) at %lu MHz\n",
 253                 stat->busy_time, stat->total_time,
 254                 DIV_ROUND_CLOSEST(stat->busy_time * 100, stat->total_time),
 255                 stat->current_frequency / USEC_PER_SEC);
 256
 257         return 0;
 258 }
 259
 260 static int sun8i_a33_mbus_hw_init(struct device *dev,
 261                                   struct sun8i_a33_mbus *priv,
 262                                   unsigned long ddr_freq)
 263 {
 264         u32 i, mbus_cr, mbus_freq_mhz;
 265
 266         /* Choose tREFI and tRFC to match the configured DRAM type. */
 267         mbus_cr = readl_relaxed(priv->reg_mbus + MBUS_CR);
 268         switch (MBUS_CR_GET_DRAM_TYPE(mbus_cr)) {
 269         case MBUS_CR_DRAM_TYPE_DDR2:
 270         case MBUS_CR_DRAM_TYPE_DDR3:
 271         case MBUS_CR_DRAM_TYPE_DDR4:
 272                 priv->tREFI_ns = 7800;
 273                 priv->tRFC_ns = 350;
 274                 break;
 275         case MBUS_CR_DRAM_TYPE_LPDDR2:
 276         case MBUS_CR_DRAM_TYPE_LPDDR3:
 277                 priv->tREFI_ns = 3900;
 278                 priv->tRFC_ns = 210;
 279                 break;
 280         default:
 281                 return -EINVAL;
 282         }
 283
 284         /* Save ODTMAP so it can be restored when raising the frequency. */
 285         priv->odtmap = readl_relaxed(priv->reg_dram + DRAM_ODTMAP);
 286
 287         /* Compute the DRAM data bus width by counting enabled DATx8 blocks. */
 288         for (i = 0; i < DRAM_DX_MAX; ++i) {
 289                 void __iomem *reg = priv->reg_dram + DRAM_DXnGCR0(i);
 290
 291                 if (!(readl_relaxed(reg) & DRAM_DXnGCR0_DXEN))
 292                         break;
 293         }
 294         priv->data_width = i;
 295
 296         dev_dbg(dev, "Detected %u-bit %sDDRx with%s ODT\n",
 297                 priv->data_width * 8,
 298                 MBUS_CR_GET_DRAM_TYPE(mbus_cr) > 4 ? "LP" : "",
 299                 priv->odtmap ? "" : "out");
 300
 301         /* Program MBUS_TMR such that the PMU period unit is microseconds. */
 302         mbus_freq_mhz = clk_get_rate(priv->clk_mbus) / USEC_PER_SEC;
 303         writel_relaxed(MBUS_TMR_PERIOD(mbus_freq_mhz),
 304                        priv->reg_mbus + MBUS_TMR);
 305
 306         /* "Master Ready Mask Register" bits must be set or MDFS will block. */
 307         writel_relaxed(0xffffffff, priv->reg_mbus + MBUS_MDFSMRMR);
 308
 309         sun8i_a33_mbus_restart_pmu_counters(priv);
 310         sun8i_a33_mbus_update_nominal_bw(priv, ddr_freq / USEC_PER_SEC);
 311
 312         return 0;
 313 }
 314
 315 static int __maybe_unused sun8i_a33_mbus_suspend(struct device *dev)
 316 {
 317         struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
 318
 319         clk_disable_unprepare(priv->clk_bus);
 320
 321         return 0;
 322 }
 323
 324 static int __maybe_unused sun8i_a33_mbus_resume(struct device *dev)
 325 {
 326         struct sun8i_a33_mbus *priv = dev_get_drvdata(dev);
 327
 328         return clk_prepare_enable(priv->clk_bus);
 329 }
 330
 331 static int sun8i_a33_mbus_probe(struct platform_device *pdev)
 332 {
 333         const struct sun8i_a33_mbus_variant *variant;
 334         struct device *dev = &pdev->dev;
 335         struct sun8i_a33_mbus *priv;
 336         unsigned long base_freq;
 337         unsigned int max_state;
 338         const char *err;
 339         int i, ret;
 340
 341         variant = device_get_match_data(dev);
 342         if (!variant)
 343                 return -EINVAL;
 344
 345         max_state = variant->max_dram_divider - variant->min_dram_divider + 1;
 346
 347         priv = devm_kzalloc(dev, struct_size(priv, freq_table, max_state), GFP_KERNEL);
 348         if (!priv)
 349                 return -ENOMEM;
 350
 351         platform_set_drvdata(pdev, priv);
 352
 353         priv->variant = variant;
 354
 355         priv->reg_dram = devm_platform_ioremap_resource_byname(pdev, "dram");
 356         if (IS_ERR(priv->reg_dram))
 357                 return PTR_ERR(priv->reg_dram);
 358
 359         priv->reg_mbus = devm_platform_ioremap_resource_byname(pdev, "mbus");
 360         if (IS_ERR(priv->reg_mbus))
 361                 return PTR_ERR(priv->reg_mbus);
 362
 363         priv->clk_bus = devm_clk_get(dev, "bus");
 364         if (IS_ERR(priv->clk_bus))
 365                 return dev_err_probe(dev, PTR_ERR(priv->clk_bus),
 366                                      "failed to get bus clock\n");
 367
 368         priv->clk_dram = devm_clk_get(dev, "dram");
 369         if (IS_ERR(priv->clk_dram))
 370                 return dev_err_probe(dev, PTR_ERR(priv->clk_dram),
 371                                      "failed to get dram clock\n");
 372
 373         priv->clk_mbus = devm_clk_get(dev, "mbus");
 374         if (IS_ERR(priv->clk_mbus))
 375                 return dev_err_probe(dev, PTR_ERR(priv->clk_mbus),
 376                                      "failed to get mbus clock\n");
 377
 378         ret = clk_prepare_enable(priv->clk_bus);
 379         if (ret)
 380                 return dev_err_probe(dev, ret,
 381                                      "failed to enable bus clock\n");
 382
 383         /* Lock the DRAM clock rate to keep priv->nominal_bw in sync. */
 384         ret = clk_rate_exclusive_get(priv->clk_dram);
 385         if (ret) {
 386                 err = "failed to lock dram clock rate\n";
 387                 goto err_disable_bus;
 388         }
 389
 390         /* Lock the MBUS clock rate to keep MBUS_TMR_PERIOD in sync. */
 391         ret = clk_rate_exclusive_get(priv->clk_mbus);
 392         if (ret) {
 393                 err = "failed to lock mbus clock rate\n";
 394                 goto err_unlock_dram;
 395         }
 396
 397         priv->gov_data.upthreshold      = 10;
 398         priv->gov_data.downdifferential =  5;
 399
 400         priv->profile.initial_freq      = clk_get_rate(priv->clk_dram);
 401         priv->profile.polling_ms        = 1000;
 402         priv->profile.target            = sun8i_a33_mbus_set_dram_target;
 403         priv->profile.get_dev_status    = sun8i_a33_mbus_get_dram_status;
 404         priv->profile.freq_table        = priv->freq_table;
 405         priv->profile.max_state         = max_state;
 406
 407         ret = devm_pm_opp_set_clkname(dev, "dram");
 408         if (ret) {
 409                 err = "failed to add OPP table\n";
 410                 goto err_unlock_mbus;
 411         }
 412
 413         base_freq = clk_get_rate(clk_get_parent(priv->clk_dram));
 414         for (i = 0; i < max_state; ++i) {
 415                 unsigned int div = variant->max_dram_divider - i;
 416
 417                 priv->freq_table[i] = base_freq / div;
 418
 419                 ret = dev_pm_opp_add(dev, priv->freq_table[i], 0);
 420                 if (ret) {
 421                         err = "failed to add OPPs\n";
 422                         goto err_remove_opps;
 423                 }
 424         }
 425
 426         ret = sun8i_a33_mbus_hw_init(dev, priv, priv->profile.initial_freq);
 427         if (ret) {
 428                 err = "failed to init hardware\n";
 429                 goto err_remove_opps;
 430         }
 431
 432         priv->devfreq_dram = devfreq_add_device(dev, &priv->profile,
 433                                                 DEVFREQ_GOV_SIMPLE_ONDEMAND,
 434                                                 &priv->gov_data);
 435         if (IS_ERR(priv->devfreq_dram)) {
 436                 ret = PTR_ERR(priv->devfreq_dram);
 437                 err = "failed to add devfreq device\n";
 438                 goto err_remove_opps;
 439         }
 440
 441         /*
 442          * This must be set manually after registering the devfreq device,
 443          * because there is no way to select a dynamic OPP as the suspend OPP.
 444          */
 445         priv->devfreq_dram->suspend_freq = priv->freq_table[0];
 446
 447         return 0;
 448
 449 err_remove_opps:
 450         dev_pm_opp_remove_all_dynamic(dev);
 451 err_unlock_mbus:
 452         clk_rate_exclusive_put(priv->clk_mbus);
 453 err_unlock_dram:
 454         clk_rate_exclusive_put(priv->clk_dram);
 455 err_disable_bus:
 456         clk_disable_unprepare(priv->clk_bus);
 457
 458         return dev_err_probe(dev, ret, err);
 459 }
 460
 461 static int sun8i_a33_mbus_remove(struct platform_device *pdev)
 462 {
 463         struct sun8i_a33_mbus *priv = platform_get_drvdata(pdev);
 464         unsigned long initial_freq = priv->profile.initial_freq;
 465         struct device *dev = &pdev->dev;
 466         int ret;
 467
 468         devfreq_remove_device(priv->devfreq_dram);
 469
 470         ret = sun8i_a33_mbus_set_dram_freq(priv, initial_freq);
 471         if (ret)
 472                 dev_warn(dev, "failed to restore DRAM frequency: %d\n", ret);
 473
 474         dev_pm_opp_remove_all_dynamic(dev);
 475         clk_rate_exclusive_put(priv->clk_mbus);
 476         clk_rate_exclusive_put(priv->clk_dram);
 477         clk_disable_unprepare(priv->clk_bus);
 478
 479         return 0;
 480 }
 481
 482 static const struct sun8i_a33_mbus_variant sun50i_a64_mbus = {
 483         .min_dram_divider       = 1,
 484         .max_dram_divider       = 4,
 485         .odt_freq_mhz           = 400,
 486 };
 487
 488 static const struct of_device_id sun8i_a33_mbus_of_match[] = {
 489         { .compatible = "allwinner,sun50i-a64-mbus", .data = &sun50i_a64_mbus },
 490         { .compatible = "allwinner,sun50i-h5-mbus", .data = &sun50i_a64_mbus },
 491         { },
 492 };
 493 MODULE_DEVICE_TABLE(of, sun8i_a33_mbus_of_match);
 494
 495 static SIMPLE_DEV_PM_OPS(sun8i_a33_mbus_pm_ops,
 496                          sun8i_a33_mbus_suspend, sun8i_a33_mbus_resume);
 497
 498 static struct platform_driver sun8i_a33_mbus_driver = {
 499         .probe  = sun8i_a33_mbus_probe,
 500         .remove = sun8i_a33_mbus_remove,
 501         .driver = {
 502                 .name           = "sun8i-a33-mbus",
 503                 .of_match_table = sun8i_a33_mbus_of_match,
 504                 .pm             = pm_ptr(&sun8i_a33_mbus_pm_ops),
 505         },
 506 };
 507 module_platform_driver(sun8i_a33_mbus_driver);
 508
 509 MODULE_AUTHOR("Samuel Holland <samuel@sholland.org>");
 510 MODULE_DESCRIPTION("Allwinner sun8i/sun50i MBUS DEVFREQ Driver");
 511 MODULE_LICENSE("GPL v2");