Commit | Line | Data |
---|---|---|
5af744eb LC |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright 2019 NXP | |
4 | */ | |
5 | ||
6 | #include <linux/module.h> | |
7 | #include <linux/device.h> | |
8 | #include <linux/of_device.h> | |
9 | #include <linux/platform_device.h> | |
10 | #include <linux/devfreq.h> | |
11 | #include <linux/pm_opp.h> | |
12 | #include <linux/clk.h> | |
13 | #include <linux/clk-provider.h> | |
14 | #include <linux/arm-smccc.h> | |
15 | ||
16 | #define IMX_SIP_DDR_DVFS 0xc2000004 | |
17 | ||
5af744eb LC |
18 | /* Query available frequencies. */ |
19 | #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10 | |
20 | #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11 | |
21 | ||
22 | /* | |
23 | * This should be in a 1:1 mapping with devicetree OPPs but | |
24 | * firmware provides additional info. | |
25 | */ | |
26 | struct imx8m_ddrc_freq { | |
27 | unsigned long rate; | |
28 | unsigned long smcarg; | |
29 | int dram_core_parent_index; | |
30 | int dram_alt_parent_index; | |
31 | int dram_apb_parent_index; | |
32 | }; | |
33 | ||
34 | /* Hardware limitation */ | |
35 | #define IMX8M_DDRC_MAX_FREQ_COUNT 4 | |
36 | ||
37 | /* | |
38 | * i.MX8M DRAM Controller clocks have the following structure (abridged): | |
39 | * | |
40 | * +----------+ |\ +------+ | |
41 | * | dram_pll |-------|M| dram_core | | | |
42 | * +----------+ |U|---------->| D | | |
43 | * /--|X| | D | | |
44 | * dram_alt_root | |/ | R | | |
45 | * | | C | | |
46 | * +---------+ | | | |
47 | * |FIX DIV/4| | | | |
48 | * +---------+ | | | |
49 | * composite: | | | | |
50 | * +----------+ | | | | |
51 | * | dram_alt |----/ | | | |
52 | * +----------+ | | | |
53 | * | dram_apb |-------------------->| | | |
54 | * +----------+ +------+ | |
55 | * | |
56 | * The dram_pll is used for higher rates and dram_alt is used for lower rates. | |
57 | * | |
58 | * Frequency switching is implemented in TF-A (via SMC call) and can change the | |
59 | * configuration of the clocks, including mux parents. The dram_alt and | |
60 | * dram_apb clocks are "imx composite" and their parent can change too. | |
61 | * | |
62 | * We need to prepare/enable the new mux parents head of switching and update | |
63 | * their information afterwards. | |
64 | */ | |
65 | struct imx8m_ddrc { | |
66 | struct devfreq_dev_profile profile; | |
67 | struct devfreq *devfreq; | |
68 | ||
69 | /* For frequency switching: */ | |
70 | struct clk *dram_core; | |
71 | struct clk *dram_pll; | |
72 | struct clk *dram_alt; | |
73 | struct clk *dram_apb; | |
74 | ||
75 | int freq_count; | |
76 | struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT]; | |
77 | }; | |
78 | ||
79 | static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv, | |
80 | unsigned long rate) | |
81 | { | |
82 | struct imx8m_ddrc_freq *freq; | |
83 | int i; | |
84 | ||
85 | /* | |
86 | * Firmware reports values in MT/s, so we round-down from Hz | |
87 | * Rounding is extra generous to ensure a match. | |
88 | */ | |
89 | rate = DIV_ROUND_CLOSEST(rate, 250000); | |
90 | for (i = 0; i < priv->freq_count; ++i) { | |
91 | freq = &priv->freq_table[i]; | |
92 | if (freq->rate == rate || | |
93 | freq->rate + 1 == rate || | |
94 | freq->rate - 1 == rate) | |
95 | return freq; | |
96 | } | |
97 | ||
98 | return NULL; | |
99 | } | |
100 | ||
101 | static void imx8m_ddrc_smc_set_freq(int target_freq) | |
102 | { | |
103 | struct arm_smccc_res res; | |
104 | u32 online_cpus = 0; | |
105 | int cpu; | |
106 | ||
107 | local_irq_disable(); | |
108 | ||
109 | for_each_online_cpu(cpu) | |
110 | online_cpus |= (1 << (cpu * 8)); | |
111 | ||
112 | /* change the ddr freqency */ | |
113 | arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus, | |
114 | 0, 0, 0, 0, 0, &res); | |
115 | ||
116 | local_irq_enable(); | |
117 | } | |
118 | ||
119 | static struct clk *clk_get_parent_by_index(struct clk *clk, int index) | |
120 | { | |
121 | struct clk_hw *hw; | |
122 | ||
123 | hw = clk_hw_get_parent_by_index(__clk_get_hw(clk), index); | |
124 | ||
125 | return hw ? hw->clk : NULL; | |
126 | } | |
127 | ||
128 | static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq) | |
129 | { | |
130 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); | |
131 | struct clk *new_dram_core_parent; | |
132 | struct clk *new_dram_alt_parent; | |
133 | struct clk *new_dram_apb_parent; | |
134 | int ret; | |
135 | ||
136 | /* | |
137 | * Fetch new parents | |
138 | * | |
139 | * new_dram_alt_parent and new_dram_apb_parent are optional but | |
140 | * new_dram_core_parent is not. | |
141 | */ | |
142 | new_dram_core_parent = clk_get_parent_by_index( | |
143 | priv->dram_core, freq->dram_core_parent_index - 1); | |
144 | if (!new_dram_core_parent) { | |
145 | dev_err(dev, "failed to fetch new dram_core parent\n"); | |
146 | return -EINVAL; | |
147 | } | |
148 | if (freq->dram_alt_parent_index) { | |
149 | new_dram_alt_parent = clk_get_parent_by_index( | |
150 | priv->dram_alt, | |
151 | freq->dram_alt_parent_index - 1); | |
152 | if (!new_dram_alt_parent) { | |
153 | dev_err(dev, "failed to fetch new dram_alt parent\n"); | |
154 | return -EINVAL; | |
155 | } | |
156 | } else | |
157 | new_dram_alt_parent = NULL; | |
158 | ||
159 | if (freq->dram_apb_parent_index) { | |
160 | new_dram_apb_parent = clk_get_parent_by_index( | |
161 | priv->dram_apb, | |
162 | freq->dram_apb_parent_index - 1); | |
163 | if (!new_dram_apb_parent) { | |
164 | dev_err(dev, "failed to fetch new dram_apb parent\n"); | |
165 | return -EINVAL; | |
166 | } | |
167 | } else | |
168 | new_dram_apb_parent = NULL; | |
169 | ||
170 | /* increase reference counts and ensure clks are ON before switch */ | |
171 | ret = clk_prepare_enable(new_dram_core_parent); | |
172 | if (ret) { | |
173 | dev_err(dev, "failed to enable new dram_core parent: %d\n", | |
174 | ret); | |
175 | goto out; | |
176 | } | |
177 | ret = clk_prepare_enable(new_dram_alt_parent); | |
178 | if (ret) { | |
179 | dev_err(dev, "failed to enable new dram_alt parent: %d\n", | |
180 | ret); | |
181 | goto out_disable_core_parent; | |
182 | } | |
183 | ret = clk_prepare_enable(new_dram_apb_parent); | |
184 | if (ret) { | |
185 | dev_err(dev, "failed to enable new dram_apb parent: %d\n", | |
186 | ret); | |
187 | goto out_disable_alt_parent; | |
188 | } | |
189 | ||
190 | imx8m_ddrc_smc_set_freq(freq->smcarg); | |
191 | ||
192 | /* update parents in clk tree after switch. */ | |
193 | ret = clk_set_parent(priv->dram_core, new_dram_core_parent); | |
194 | if (ret) | |
195 | dev_warn(dev, "failed to set dram_core parent: %d\n", ret); | |
196 | if (new_dram_alt_parent) { | |
197 | ret = clk_set_parent(priv->dram_alt, new_dram_alt_parent); | |
198 | if (ret) | |
199 | dev_warn(dev, "failed to set dram_alt parent: %d\n", | |
200 | ret); | |
201 | } | |
202 | if (new_dram_apb_parent) { | |
203 | ret = clk_set_parent(priv->dram_apb, new_dram_apb_parent); | |
204 | if (ret) | |
205 | dev_warn(dev, "failed to set dram_apb parent: %d\n", | |
206 | ret); | |
207 | } | |
208 | ||
209 | /* | |
210 | * Explicitly refresh dram PLL rate. | |
211 | * | |
212 | * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be | |
213 | * automatically refreshed when clk_get_rate is called on children. | |
214 | */ | |
215 | clk_get_rate(priv->dram_pll); | |
216 | ||
217 | /* | |
218 | * clk_set_parent transfer the reference count from old parent. | |
219 | * now we drop extra reference counts used during the switch | |
220 | */ | |
221 | clk_disable_unprepare(new_dram_apb_parent); | |
222 | out_disable_alt_parent: | |
223 | clk_disable_unprepare(new_dram_alt_parent); | |
224 | out_disable_core_parent: | |
225 | clk_disable_unprepare(new_dram_core_parent); | |
226 | out: | |
227 | return ret; | |
228 | } | |
229 | ||
230 | static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags) | |
231 | { | |
232 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); | |
233 | struct imx8m_ddrc_freq *freq_info; | |
234 | struct dev_pm_opp *new_opp; | |
235 | unsigned long old_freq, new_freq; | |
236 | int ret; | |
237 | ||
238 | new_opp = devfreq_recommended_opp(dev, freq, flags); | |
239 | if (IS_ERR(new_opp)) { | |
240 | ret = PTR_ERR(new_opp); | |
241 | dev_err(dev, "failed to get recommended opp: %d\n", ret); | |
242 | return ret; | |
243 | } | |
244 | dev_pm_opp_put(new_opp); | |
245 | ||
246 | old_freq = clk_get_rate(priv->dram_core); | |
247 | if (*freq == old_freq) | |
248 | return 0; | |
249 | ||
250 | freq_info = imx8m_ddrc_find_freq(priv, *freq); | |
251 | if (!freq_info) | |
252 | return -EINVAL; | |
253 | ||
254 | /* | |
255 | * Read back the clk rate to verify switch was correct and so that | |
256 | * we can report it on all error paths. | |
257 | */ | |
258 | ret = imx8m_ddrc_set_freq(dev, freq_info); | |
259 | ||
260 | new_freq = clk_get_rate(priv->dram_core); | |
261 | if (ret) | |
262 | dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n", | |
263 | *freq, old_freq, ret, new_freq); | |
264 | else if (*freq != new_freq) | |
265 | dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n", | |
266 | *freq, old_freq, new_freq); | |
267 | else | |
268 | dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n", | |
269 | *freq, old_freq); | |
270 | ||
271 | return ret; | |
272 | } | |
273 | ||
274 | static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq) | |
275 | { | |
276 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); | |
277 | ||
278 | *freq = clk_get_rate(priv->dram_core); | |
279 | ||
280 | return 0; | |
281 | } | |
282 | ||
5af744eb LC |
283 | static int imx8m_ddrc_init_freq_info(struct device *dev) |
284 | { | |
285 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); | |
286 | struct arm_smccc_res res; | |
287 | int index; | |
288 | ||
289 | /* An error here means DDR DVFS API not supported by firmware */ | |
290 | arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT, | |
291 | 0, 0, 0, 0, 0, 0, &res); | |
292 | priv->freq_count = res.a0; | |
293 | if (priv->freq_count <= 0 || | |
294 | priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT) | |
295 | return -ENODEV; | |
296 | ||
297 | for (index = 0; index < priv->freq_count; ++index) { | |
298 | struct imx8m_ddrc_freq *freq = &priv->freq_table[index]; | |
299 | ||
300 | arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO, | |
301 | index, 0, 0, 0, 0, 0, &res); | |
302 | /* Result should be strictly positive */ | |
303 | if ((long)res.a0 <= 0) | |
304 | return -ENODEV; | |
305 | ||
306 | freq->rate = res.a0; | |
307 | freq->smcarg = index; | |
308 | freq->dram_core_parent_index = res.a1; | |
309 | freq->dram_alt_parent_index = res.a2; | |
310 | freq->dram_apb_parent_index = res.a3; | |
311 | ||
312 | /* dram_core has 2 options: dram_pll or dram_alt_root */ | |
313 | if (freq->dram_core_parent_index != 1 && | |
314 | freq->dram_core_parent_index != 2) | |
315 | return -ENODEV; | |
316 | /* dram_apb and dram_alt have exactly 8 possible parents */ | |
317 | if (freq->dram_alt_parent_index > 8 || | |
318 | freq->dram_apb_parent_index > 8) | |
319 | return -ENODEV; | |
320 | /* dram_core from alt requires explicit dram_alt parent */ | |
321 | if (freq->dram_core_parent_index == 2 && | |
322 | freq->dram_alt_parent_index == 0) | |
323 | return -ENODEV; | |
324 | } | |
325 | ||
326 | return 0; | |
327 | } | |
328 | ||
329 | static int imx8m_ddrc_check_opps(struct device *dev) | |
330 | { | |
331 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); | |
332 | struct imx8m_ddrc_freq *freq_info; | |
333 | struct dev_pm_opp *opp; | |
334 | unsigned long freq; | |
335 | int i, opp_count; | |
336 | ||
337 | /* Enumerate DT OPPs and disable those not supported by firmware */ | |
338 | opp_count = dev_pm_opp_get_opp_count(dev); | |
339 | if (opp_count < 0) | |
340 | return opp_count; | |
341 | for (i = 0, freq = 0; i < opp_count; ++i, ++freq) { | |
342 | opp = dev_pm_opp_find_freq_ceil(dev, &freq); | |
343 | if (IS_ERR(opp)) { | |
344 | dev_err(dev, "Failed enumerating OPPs: %ld\n", | |
345 | PTR_ERR(opp)); | |
346 | return PTR_ERR(opp); | |
347 | } | |
348 | dev_pm_opp_put(opp); | |
349 | ||
350 | freq_info = imx8m_ddrc_find_freq(priv, freq); | |
351 | if (!freq_info) { | |
352 | dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n", | |
353 | freq, DIV_ROUND_CLOSEST(freq, 250000)); | |
354 | dev_pm_opp_disable(dev, freq); | |
355 | } | |
356 | } | |
357 | ||
358 | return 0; | |
359 | } | |
360 | ||
361 | static void imx8m_ddrc_exit(struct device *dev) | |
362 | { | |
363 | dev_pm_opp_of_remove_table(dev); | |
364 | } | |
365 | ||
366 | static int imx8m_ddrc_probe(struct platform_device *pdev) | |
367 | { | |
368 | struct device *dev = &pdev->dev; | |
369 | struct imx8m_ddrc *priv; | |
370 | const char *gov = DEVFREQ_GOV_USERSPACE; | |
371 | int ret; | |
372 | ||
373 | priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); | |
374 | if (!priv) | |
375 | return -ENOMEM; | |
376 | ||
377 | platform_set_drvdata(pdev, priv); | |
378 | ||
379 | ret = imx8m_ddrc_init_freq_info(dev); | |
380 | if (ret) { | |
381 | dev_err(dev, "failed to init firmware freq info: %d\n", ret); | |
382 | return ret; | |
383 | } | |
384 | ||
385 | priv->dram_core = devm_clk_get(dev, "core"); | |
10800fec Y |
386 | if (IS_ERR(priv->dram_core)) { |
387 | ret = PTR_ERR(priv->dram_core); | |
388 | dev_err(dev, "failed to fetch core clock: %d\n", ret); | |
389 | return ret; | |
390 | } | |
5af744eb | 391 | priv->dram_pll = devm_clk_get(dev, "pll"); |
10800fec Y |
392 | if (IS_ERR(priv->dram_pll)) { |
393 | ret = PTR_ERR(priv->dram_pll); | |
394 | dev_err(dev, "failed to fetch pll clock: %d\n", ret); | |
395 | return ret; | |
396 | } | |
5af744eb | 397 | priv->dram_alt = devm_clk_get(dev, "alt"); |
10800fec Y |
398 | if (IS_ERR(priv->dram_alt)) { |
399 | ret = PTR_ERR(priv->dram_alt); | |
400 | dev_err(dev, "failed to fetch alt clock: %d\n", ret); | |
401 | return ret; | |
402 | } | |
5af744eb | 403 | priv->dram_apb = devm_clk_get(dev, "apb"); |
10800fec Y |
404 | if (IS_ERR(priv->dram_apb)) { |
405 | ret = PTR_ERR(priv->dram_apb); | |
406 | dev_err(dev, "failed to fetch apb clock: %d\n", ret); | |
5af744eb LC |
407 | return ret; |
408 | } | |
409 | ||
410 | ret = dev_pm_opp_of_add_table(dev); | |
411 | if (ret < 0) { | |
412 | dev_err(dev, "failed to get OPP table\n"); | |
413 | return ret; | |
414 | } | |
415 | ||
416 | ret = imx8m_ddrc_check_opps(dev); | |
417 | if (ret < 0) | |
418 | goto err; | |
419 | ||
5af744eb | 420 | priv->profile.target = imx8m_ddrc_target; |
5af744eb LC |
421 | priv->profile.exit = imx8m_ddrc_exit; |
422 | priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq; | |
423 | priv->profile.initial_freq = clk_get_rate(priv->dram_core); | |
424 | ||
425 | priv->devfreq = devm_devfreq_add_device(dev, &priv->profile, | |
426 | gov, NULL); | |
427 | if (IS_ERR(priv->devfreq)) { | |
428 | ret = PTR_ERR(priv->devfreq); | |
429 | dev_err(dev, "failed to add devfreq device: %d\n", ret); | |
430 | goto err; | |
431 | } | |
432 | ||
433 | return 0; | |
434 | ||
435 | err: | |
436 | dev_pm_opp_of_remove_table(dev); | |
437 | return ret; | |
438 | } | |
439 | ||
440 | static const struct of_device_id imx8m_ddrc_of_match[] = { | |
441 | { .compatible = "fsl,imx8m-ddrc", }, | |
442 | { /* sentinel */ }, | |
443 | }; | |
444 | MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match); | |
445 | ||
446 | static struct platform_driver imx8m_ddrc_platdrv = { | |
447 | .probe = imx8m_ddrc_probe, | |
448 | .driver = { | |
449 | .name = "imx8m-ddrc-devfreq", | |
0a7dc831 | 450 | .of_match_table = imx8m_ddrc_of_match, |
5af744eb LC |
451 | }, |
452 | }; | |
453 | module_platform_driver(imx8m_ddrc_platdrv); | |
454 | ||
455 | MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver"); | |
456 | MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>"); | |
457 | MODULE_LICENSE("GPL v2"); |