Commit | Line | Data |
---|---|---|
2849dd8b TD |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2018, The Linux Foundation. All rights reserved. | |
4 | */ | |
5 | ||
6 | #include <linux/bitfield.h> | |
7 | #include <linux/cpufreq.h> | |
8 | #include <linux/init.h> | |
51c843cf | 9 | #include <linux/interconnect.h> |
275157b3 | 10 | #include <linux/interrupt.h> |
2849dd8b TD |
11 | #include <linux/kernel.h> |
12 | #include <linux/module.h> | |
13 | #include <linux/of_address.h> | |
14 | #include <linux/of_platform.h> | |
55538fbc | 15 | #include <linux/pm_opp.h> |
2849dd8b | 16 | #include <linux/slab.h> |
275157b3 | 17 | #include <linux/spinlock.h> |
2849dd8b TD |
18 | |
19 | #define LUT_MAX_ENTRIES 40U | |
20 | #define LUT_SRC GENMASK(31, 30) | |
21 | #define LUT_L_VAL GENMASK(7, 0) | |
22 | #define LUT_CORE_COUNT GENMASK(18, 16) | |
55538fbc | 23 | #define LUT_VOLT GENMASK(11, 0) |
2849dd8b | 24 | #define CLK_HW_DIV 2 |
0eae1e37 | 25 | #define LUT_TURBO_IND 1 |
2849dd8b | 26 | |
e4e64486 VZ |
27 | #define GT_IRQ_STATUS BIT(2) |
28 | ||
275157b3 TG |
29 | #define HZ_PER_KHZ 1000 |
30 | ||
dcd1fd72 MS |
31 | struct qcom_cpufreq_soc_data { |
32 | u32 reg_enable; | |
f84ccad5 | 33 | u32 reg_domain_state; |
c377d4ba | 34 | u32 reg_dcvs_ctrl; |
dcd1fd72 MS |
35 | u32 reg_freq_lut; |
36 | u32 reg_volt_lut; | |
e4e64486 | 37 | u32 reg_intr_clr; |
275157b3 | 38 | u32 reg_current_vote; |
dcd1fd72 MS |
39 | u32 reg_perf_state; |
40 | u8 lut_row_size; | |
41 | }; | |
42 | ||
43 | struct qcom_cpufreq_data { | |
44 | void __iomem *base; | |
67fc209b | 45 | struct resource *res; |
dcd1fd72 | 46 | const struct qcom_cpufreq_soc_data *soc_data; |
275157b3 TG |
47 | |
48 | /* | |
49 | * Mutex to synchronize between de-init sequence and re-starting LMh | |
50 | * polling/interrupts | |
51 | */ | |
52 | struct mutex throttle_lock; | |
53 | int throttle_irq; | |
be6592ed | 54 | char irq_name[15]; |
275157b3 TG |
55 | bool cancel_throttle; |
56 | struct delayed_work throttle_work; | |
57 | struct cpufreq_policy *policy; | |
c377d4ba BA |
58 | |
59 | bool per_core_dcvs; | |
dcd1fd72 | 60 | }; |
2849dd8b TD |
61 | |
62 | static unsigned long cpu_hw_rate, xo_rate; | |
51c843cf SS |
63 | static bool icc_scaling_enabled; |
64 | ||
65 | static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, | |
66 | unsigned long freq_khz) | |
67 | { | |
68 | unsigned long freq_hz = freq_khz * 1000; | |
69 | struct dev_pm_opp *opp; | |
70 | struct device *dev; | |
71 | int ret; | |
72 | ||
73 | dev = get_cpu_device(policy->cpu); | |
74 | if (!dev) | |
75 | return -ENODEV; | |
76 | ||
77 | opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true); | |
78 | if (IS_ERR(opp)) | |
79 | return PTR_ERR(opp); | |
80 | ||
8d25157f | 81 | ret = dev_pm_opp_set_opp(dev, opp); |
51c843cf SS |
82 | dev_pm_opp_put(opp); |
83 | return ret; | |
84 | } | |
85 | ||
86 | static int qcom_cpufreq_update_opp(struct device *cpu_dev, | |
87 | unsigned long freq_khz, | |
88 | unsigned long volt) | |
89 | { | |
90 | unsigned long freq_hz = freq_khz * 1000; | |
91 | int ret; | |
92 | ||
93 | /* Skip voltage update if the opp table is not available */ | |
94 | if (!icc_scaling_enabled) | |
95 | return dev_pm_opp_add(cpu_dev, freq_hz, volt); | |
96 | ||
97 | ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt); | |
98 | if (ret) { | |
99 | dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz); | |
100 | return ret; | |
101 | } | |
102 | ||
103 | return dev_pm_opp_enable(cpu_dev, freq_hz); | |
104 | } | |
2849dd8b TD |
105 | |
106 | static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, | |
107 | unsigned int index) | |
108 | { | |
dcd1fd72 MS |
109 | struct qcom_cpufreq_data *data = policy->driver_data; |
110 | const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; | |
ada54f35 | 111 | unsigned long freq = policy->freq_table[index].frequency; |
c377d4ba | 112 | unsigned int i; |
2849dd8b | 113 | |
dcd1fd72 | 114 | writel_relaxed(index, data->base + soc_data->reg_perf_state); |
2849dd8b | 115 | |
c377d4ba BA |
116 | if (data->per_core_dcvs) |
117 | for (i = 1; i < cpumask_weight(policy->related_cpus); i++) | |
118 | writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); | |
119 | ||
51c843cf SS |
120 | if (icc_scaling_enabled) |
121 | qcom_cpufreq_set_bw(policy, freq); | |
122 | ||
2849dd8b TD |
123 | return 0; |
124 | } | |
125 | ||
126 | static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) | |
127 | { | |
dcd1fd72 MS |
128 | struct qcom_cpufreq_data *data; |
129 | const struct qcom_cpufreq_soc_data *soc_data; | |
2849dd8b TD |
130 | struct cpufreq_policy *policy; |
131 | unsigned int index; | |
132 | ||
133 | policy = cpufreq_cpu_get_raw(cpu); | |
134 | if (!policy) | |
135 | return 0; | |
136 | ||
dcd1fd72 MS |
137 | data = policy->driver_data; |
138 | soc_data = data->soc_data; | |
2849dd8b | 139 | |
dcd1fd72 | 140 | index = readl_relaxed(data->base + soc_data->reg_perf_state); |
2849dd8b TD |
141 | index = min(index, LUT_MAX_ENTRIES - 1); |
142 | ||
143 | return policy->freq_table[index].frequency; | |
144 | } | |
145 | ||
146 | static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, | |
147 | unsigned int target_freq) | |
148 | { | |
dcd1fd72 MS |
149 | struct qcom_cpufreq_data *data = policy->driver_data; |
150 | const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; | |
292072c3 | 151 | unsigned int index; |
c377d4ba | 152 | unsigned int i; |
2849dd8b TD |
153 | |
154 | index = policy->cached_resolved_idx; | |
dcd1fd72 | 155 | writel_relaxed(index, data->base + soc_data->reg_perf_state); |
2849dd8b | 156 | |
c377d4ba BA |
157 | if (data->per_core_dcvs) |
158 | for (i = 1; i < cpumask_weight(policy->related_cpus); i++) | |
159 | writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); | |
160 | ||
1a0419b0 | 161 | return policy->freq_table[index].frequency; |
2849dd8b TD |
162 | } |
163 | ||
55538fbc | 164 | static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, |
dcd1fd72 | 165 | struct cpufreq_policy *policy) |
2849dd8b | 166 | { |
0eae1e37 | 167 | u32 data, src, lval, i, core_count, prev_freq = 0, freq; |
55538fbc | 168 | u32 volt; |
2849dd8b | 169 | struct cpufreq_frequency_table *table; |
51c843cf SS |
170 | struct dev_pm_opp *opp; |
171 | unsigned long rate; | |
172 | int ret; | |
dcd1fd72 MS |
173 | struct qcom_cpufreq_data *drv_data = policy->driver_data; |
174 | const struct qcom_cpufreq_soc_data *soc_data = drv_data->soc_data; | |
2849dd8b TD |
175 | |
176 | table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); | |
177 | if (!table) | |
178 | return -ENOMEM; | |
179 | ||
51c843cf SS |
180 | ret = dev_pm_opp_of_add_table(cpu_dev); |
181 | if (!ret) { | |
182 | /* Disable all opps and cross-validate against LUT later */ | |
183 | icc_scaling_enabled = true; | |
184 | for (rate = 0; ; rate++) { | |
185 | opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); | |
186 | if (IS_ERR(opp)) | |
187 | break; | |
188 | ||
189 | dev_pm_opp_put(opp); | |
190 | dev_pm_opp_disable(cpu_dev, rate); | |
191 | } | |
192 | } else if (ret != -ENODEV) { | |
193 | dev_err(cpu_dev, "Invalid opp table in device tree\n"); | |
194 | return ret; | |
195 | } else { | |
afdb219b | 196 | policy->fast_switch_possible = true; |
51c843cf SS |
197 | icc_scaling_enabled = false; |
198 | } | |
199 | ||
2849dd8b | 200 | for (i = 0; i < LUT_MAX_ENTRIES; i++) { |
dcd1fd72 MS |
201 | data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + |
202 | i * soc_data->lut_row_size); | |
2849dd8b TD |
203 | src = FIELD_GET(LUT_SRC, data); |
204 | lval = FIELD_GET(LUT_L_VAL, data); | |
205 | core_count = FIELD_GET(LUT_CORE_COUNT, data); | |
206 | ||
dcd1fd72 MS |
207 | data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + |
208 | i * soc_data->lut_row_size); | |
55538fbc TD |
209 | volt = FIELD_GET(LUT_VOLT, data) * 1000; |
210 | ||
2849dd8b TD |
211 | if (src) |
212 | freq = xo_rate * lval / 1000; | |
213 | else | |
214 | freq = cpu_hw_rate / 1000; | |
215 | ||
0eae1e37 | 216 | if (freq != prev_freq && core_count != LUT_TURBO_IND) { |
bc9b9c5a MK |
217 | if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { |
218 | table[i].frequency = freq; | |
219 | dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, | |
2849dd8b | 220 | freq, core_count); |
bc9b9c5a MK |
221 | } else { |
222 | dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); | |
223 | table[i].frequency = CPUFREQ_ENTRY_INVALID; | |
224 | } | |
225 | ||
0eae1e37 | 226 | } else if (core_count == LUT_TURBO_IND) { |
55538fbc | 227 | table[i].frequency = CPUFREQ_ENTRY_INVALID; |
2849dd8b TD |
228 | } |
229 | ||
230 | /* | |
231 | * Two of the same frequencies with the same core counts means | |
232 | * end of table | |
233 | */ | |
0eae1e37 | 234 | if (i > 0 && prev_freq == freq) { |
2849dd8b TD |
235 | struct cpufreq_frequency_table *prev = &table[i - 1]; |
236 | ||
237 | /* | |
238 | * Only treat the last frequency that might be a boost | |
239 | * as the boost frequency | |
240 | */ | |
0eae1e37 | 241 | if (prev->frequency == CPUFREQ_ENTRY_INVALID) { |
bc9b9c5a MK |
242 | if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { |
243 | prev->frequency = prev_freq; | |
244 | prev->flags = CPUFREQ_BOOST_FREQ; | |
245 | } else { | |
246 | dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", | |
247 | freq); | |
248 | } | |
2849dd8b TD |
249 | } |
250 | ||
251 | break; | |
252 | } | |
253 | ||
2849dd8b TD |
254 | prev_freq = freq; |
255 | } | |
256 | ||
257 | table[i].frequency = CPUFREQ_TABLE_END; | |
258 | policy->freq_table = table; | |
55538fbc | 259 | dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); |
2849dd8b TD |
260 | |
261 | return 0; | |
262 | } | |
263 | ||
264 | static void qcom_get_related_cpus(int index, struct cpumask *m) | |
265 | { | |
266 | struct device_node *cpu_np; | |
267 | struct of_phandle_args args; | |
268 | int cpu, ret; | |
269 | ||
270 | for_each_possible_cpu(cpu) { | |
271 | cpu_np = of_cpu_device_node_get(cpu); | |
272 | if (!cpu_np) | |
273 | continue; | |
274 | ||
275 | ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", | |
276 | "#freq-domain-cells", 0, | |
277 | &args); | |
278 | of_node_put(cpu_np); | |
279 | if (ret < 0) | |
280 | continue; | |
281 | ||
282 | if (index == args.args[0]) | |
283 | cpumask_set_cpu(cpu, m); | |
284 | } | |
285 | } | |
286 | ||
f84ccad5 | 287 | static unsigned long qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data) |
275157b3 | 288 | { |
f84ccad5 | 289 | unsigned int lval; |
275157b3 | 290 | |
f84ccad5 VZ |
291 | if (data->soc_data->reg_current_vote) |
292 | lval = readl_relaxed(data->base + data->soc_data->reg_current_vote) & 0x3ff; | |
293 | else | |
294 | lval = readl_relaxed(data->base + data->soc_data->reg_domain_state) & 0xff; | |
295 | ||
296 | return lval * xo_rate; | |
275157b3 TG |
297 | } |
298 | ||
299 | static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) | |
300 | { | |
275157b3 | 301 | struct cpufreq_policy *policy = data->policy; |
5e4f009d | 302 | int cpu = cpumask_first(policy->related_cpus); |
275157b3 | 303 | struct device *dev = get_cpu_device(cpu); |
0258cb19 | 304 | unsigned long freq_hz, throttled_freq; |
275157b3 | 305 | struct dev_pm_opp *opp; |
275157b3 TG |
306 | |
307 | /* | |
308 | * Get the h/w throttled frequency, normalize it using the | |
309 | * registered opp table and use it to calculate thermal pressure. | |
310 | */ | |
f84ccad5 | 311 | freq_hz = qcom_lmh_get_throttle_freq(data); |
275157b3 TG |
312 | |
313 | opp = dev_pm_opp_find_freq_floor(dev, &freq_hz); | |
314 | if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) | |
6240aaad | 315 | opp = dev_pm_opp_find_freq_ceil(dev, &freq_hz); |
275157b3 | 316 | |
6240aaad DB |
317 | if (IS_ERR(opp)) { |
318 | dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp); | |
319 | } else { | |
320 | throttled_freq = freq_hz / HZ_PER_KHZ; | |
321 | ||
322 | /* Update thermal pressure (the boost frequencies are accepted) */ | |
323 | arch_update_thermal_pressure(policy->related_cpus, throttled_freq); | |
275157b3 | 324 | |
6240aaad DB |
325 | dev_pm_opp_put(opp); |
326 | } | |
275157b3 TG |
327 | |
328 | /* | |
329 | * In the unlikely case policy is unregistered do not enable | |
330 | * polling or h/w interrupt | |
331 | */ | |
332 | mutex_lock(&data->throttle_lock); | |
333 | if (data->cancel_throttle) | |
334 | goto out; | |
335 | ||
336 | /* | |
337 | * If h/w throttled frequency is higher than what cpufreq has requested | |
338 | * for, then stop polling and switch back to interrupt mechanism. | |
339 | */ | |
340 | if (throttled_freq >= qcom_cpufreq_hw_get(cpu)) | |
341 | enable_irq(data->throttle_irq); | |
342 | else | |
343 | mod_delayed_work(system_highpri_wq, &data->throttle_work, | |
344 | msecs_to_jiffies(10)); | |
345 | ||
346 | out: | |
347 | mutex_unlock(&data->throttle_lock); | |
348 | } | |
349 | ||
350 | static void qcom_lmh_dcvs_poll(struct work_struct *work) | |
351 | { | |
352 | struct qcom_cpufreq_data *data; | |
353 | ||
354 | data = container_of(work, struct qcom_cpufreq_data, throttle_work.work); | |
355 | qcom_lmh_dcvs_notify(data); | |
356 | } | |
357 | ||
358 | static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data) | |
359 | { | |
360 | struct qcom_cpufreq_data *c_data = data; | |
361 | ||
362 | /* Disable interrupt and enable polling */ | |
363 | disable_irq_nosync(c_data->throttle_irq); | |
e0e27c3d | 364 | schedule_delayed_work(&c_data->throttle_work, 0); |
275157b3 | 365 | |
e4e64486 VZ |
366 | if (c_data->soc_data->reg_intr_clr) |
367 | writel_relaxed(GT_IRQ_STATUS, | |
368 | c_data->base + c_data->soc_data->reg_intr_clr); | |
369 | ||
e0e27c3d | 370 | return IRQ_HANDLED; |
275157b3 TG |
371 | } |
372 | ||
dcd1fd72 MS |
373 | static const struct qcom_cpufreq_soc_data qcom_soc_data = { |
374 | .reg_enable = 0x0, | |
c377d4ba | 375 | .reg_dcvs_ctrl = 0xbc, |
dcd1fd72 MS |
376 | .reg_freq_lut = 0x110, |
377 | .reg_volt_lut = 0x114, | |
275157b3 | 378 | .reg_current_vote = 0x704, |
dcd1fd72 MS |
379 | .reg_perf_state = 0x920, |
380 | .lut_row_size = 32, | |
381 | }; | |
382 | ||
49b59f4c MS |
383 | static const struct qcom_cpufreq_soc_data epss_soc_data = { |
384 | .reg_enable = 0x0, | |
f84ccad5 | 385 | .reg_domain_state = 0x20, |
c377d4ba | 386 | .reg_dcvs_ctrl = 0xb0, |
49b59f4c MS |
387 | .reg_freq_lut = 0x100, |
388 | .reg_volt_lut = 0x200, | |
e4e64486 | 389 | .reg_intr_clr = 0x308, |
49b59f4c MS |
390 | .reg_perf_state = 0x320, |
391 | .lut_row_size = 4, | |
392 | }; | |
393 | ||
dcd1fd72 MS |
394 | static const struct of_device_id qcom_cpufreq_hw_match[] = { |
395 | { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, | |
49b59f4c | 396 | { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, |
dcd1fd72 MS |
397 | {} |
398 | }; | |
399 | MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); | |
400 | ||
275157b3 TG |
401 | static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) |
402 | { | |
403 | struct qcom_cpufreq_data *data = policy->driver_data; | |
404 | struct platform_device *pdev = cpufreq_get_driver_data(); | |
275157b3 TG |
405 | int ret; |
406 | ||
407 | /* | |
408 | * Look for LMh interrupt. If no interrupt line is specified / | |
409 | * if there is an error, allow cpufreq to be enabled as usual. | |
410 | */ | |
8f5783ad SB |
411 | data->throttle_irq = platform_get_irq_optional(pdev, index); |
412 | if (data->throttle_irq == -ENXIO) | |
413 | return 0; | |
414 | if (data->throttle_irq < 0) | |
415 | return data->throttle_irq; | |
275157b3 TG |
416 | |
417 | data->cancel_throttle = false; | |
418 | data->policy = policy; | |
419 | ||
420 | mutex_init(&data->throttle_lock); | |
421 | INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll); | |
422 | ||
be6592ed | 423 | snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu); |
275157b3 | 424 | ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq, |
ef8ee1cb | 425 | IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data); |
275157b3 | 426 | if (ret) { |
be6592ed | 427 | dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret); |
275157b3 TG |
428 | return 0; |
429 | } | |
430 | ||
3ed6dfbd VZ |
431 | ret = irq_set_affinity_hint(data->throttle_irq, policy->cpus); |
432 | if (ret) | |
433 | dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", | |
434 | data->irq_name, data->throttle_irq); | |
435 | ||
275157b3 TG |
436 | return 0; |
437 | } | |
438 | ||
a1eb080a | 439 | static int qcom_cpufreq_hw_cpu_online(struct cpufreq_policy *policy) |
275157b3 | 440 | { |
a1eb080a DB |
441 | struct qcom_cpufreq_data *data = policy->driver_data; |
442 | struct platform_device *pdev = cpufreq_get_driver_data(); | |
443 | int ret; | |
444 | ||
668a7a12 SB |
445 | if (data->throttle_irq <= 0) |
446 | return 0; | |
447 | ||
a1eb080a DB |
448 | ret = irq_set_affinity_hint(data->throttle_irq, policy->cpus); |
449 | if (ret) | |
450 | dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", | |
451 | data->irq_name, data->throttle_irq); | |
452 | ||
453 | return ret; | |
454 | } | |
455 | ||
456 | static int qcom_cpufreq_hw_cpu_offline(struct cpufreq_policy *policy) | |
457 | { | |
458 | struct qcom_cpufreq_data *data = policy->driver_data; | |
459 | ||
275157b3 | 460 | if (data->throttle_irq <= 0) |
a1eb080a | 461 | return 0; |
275157b3 TG |
462 | |
463 | mutex_lock(&data->throttle_lock); | |
464 | data->cancel_throttle = true; | |
465 | mutex_unlock(&data->throttle_lock); | |
466 | ||
467 | cancel_delayed_work_sync(&data->throttle_work); | |
be5985b3 | 468 | irq_set_affinity_hint(data->throttle_irq, NULL); |
a1eb080a DB |
469 | |
470 | return 0; | |
471 | } | |
472 | ||
473 | static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) | |
474 | { | |
668a7a12 SB |
475 | if (data->throttle_irq <= 0) |
476 | return; | |
477 | ||
275157b3 TG |
478 | free_irq(data->throttle_irq, data); |
479 | } | |
480 | ||
2849dd8b TD |
481 | static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) |
482 | { | |
bd74e286 MS |
483 | struct platform_device *pdev = cpufreq_get_driver_data(); |
484 | struct device *dev = &pdev->dev; | |
2849dd8b TD |
485 | struct of_phandle_args args; |
486 | struct device_node *cpu_np; | |
55538fbc | 487 | struct device *cpu_dev; |
67fc209b | 488 | struct resource *res; |
2849dd8b | 489 | void __iomem *base; |
dcd1fd72 | 490 | struct qcom_cpufreq_data *data; |
2849dd8b TD |
491 | int ret, index; |
492 | ||
55538fbc TD |
493 | cpu_dev = get_cpu_device(policy->cpu); |
494 | if (!cpu_dev) { | |
495 | pr_err("%s: failed to get cpu%d device\n", __func__, | |
496 | policy->cpu); | |
497 | return -ENODEV; | |
498 | } | |
499 | ||
2849dd8b TD |
500 | cpu_np = of_cpu_device_node_get(policy->cpu); |
501 | if (!cpu_np) | |
502 | return -EINVAL; | |
503 | ||
504 | ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", | |
505 | "#freq-domain-cells", 0, &args); | |
506 | of_node_put(cpu_np); | |
507 | if (ret) | |
508 | return ret; | |
509 | ||
510 | index = args.args[0]; | |
511 | ||
67fc209b SG |
512 | res = platform_get_resource(pdev, IORESOURCE_MEM, index); |
513 | if (!res) { | |
514 | dev_err(dev, "failed to get mem resource %d\n", index); | |
515 | return -ENODEV; | |
516 | } | |
517 | ||
518 | if (!request_mem_region(res->start, resource_size(res), res->name)) { | |
519 | dev_err(dev, "failed to request resource %pR\n", res); | |
520 | return -EBUSY; | |
521 | } | |
2849dd8b | 522 | |
67fc209b | 523 | base = ioremap(res->start, resource_size(res)); |
536eb97a | 524 | if (!base) { |
67fc209b | 525 | dev_err(dev, "failed to map resource %pR\n", res); |
536eb97a | 526 | ret = -ENOMEM; |
67fc209b SG |
527 | goto release_region; |
528 | } | |
529 | ||
530 | data = kzalloc(sizeof(*data), GFP_KERNEL); | |
dcd1fd72 MS |
531 | if (!data) { |
532 | ret = -ENOMEM; | |
67fc209b | 533 | goto unmap_base; |
dcd1fd72 MS |
534 | } |
535 | ||
536 | data->soc_data = of_device_get_match_data(&pdev->dev); | |
537 | data->base = base; | |
67fc209b | 538 | data->res = res; |
2849dd8b TD |
539 | |
540 | /* HW should be in enabled state to proceed */ | |
dcd1fd72 | 541 | if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { |
2849dd8b TD |
542 | dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); |
543 | ret = -ENODEV; | |
544 | goto error; | |
545 | } | |
546 | ||
c377d4ba BA |
547 | if (readl_relaxed(base + data->soc_data->reg_dcvs_ctrl) & 0x1) |
548 | data->per_core_dcvs = true; | |
549 | ||
2849dd8b | 550 | qcom_get_related_cpus(index, policy->cpus); |
b48cd0d1 | 551 | if (cpumask_empty(policy->cpus)) { |
2849dd8b TD |
552 | dev_err(dev, "Domain-%d failed to get related CPUs\n", index); |
553 | ret = -ENOENT; | |
554 | goto error; | |
555 | } | |
556 | ||
dcd1fd72 | 557 | policy->driver_data = data; |
f0712ace | 558 | policy->dvfs_possible_from_any_cpu = true; |
2849dd8b | 559 | |
dcd1fd72 | 560 | ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); |
2849dd8b TD |
561 | if (ret) { |
562 | dev_err(dev, "Domain-%d failed to read LUT\n", index); | |
563 | goto error; | |
564 | } | |
565 | ||
55538fbc TD |
566 | ret = dev_pm_opp_get_opp_count(cpu_dev); |
567 | if (ret <= 0) { | |
568 | dev_err(cpu_dev, "Failed to add OPPs\n"); | |
569 | ret = -ENODEV; | |
570 | goto error; | |
571 | } | |
572 | ||
26699172 SG |
573 | if (policy_has_boost_freq(policy)) { |
574 | ret = cpufreq_enable_boost_support(); | |
575 | if (ret) | |
576 | dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); | |
577 | } | |
578 | ||
275157b3 TG |
579 | ret = qcom_cpufreq_hw_lmh_init(policy, index); |
580 | if (ret) | |
581 | goto error; | |
582 | ||
2849dd8b TD |
583 | return 0; |
584 | error: | |
67fc209b SG |
585 | kfree(data); |
586 | unmap_base: | |
02fc4095 | 587 | iounmap(base); |
67fc209b SG |
588 | release_region: |
589 | release_mem_region(res->start, resource_size(res)); | |
2849dd8b TD |
590 | return ret; |
591 | } | |
592 | ||
593 | static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) | |
594 | { | |
55538fbc | 595 | struct device *cpu_dev = get_cpu_device(policy->cpu); |
dcd1fd72 | 596 | struct qcom_cpufreq_data *data = policy->driver_data; |
67fc209b SG |
597 | struct resource *res = data->res; |
598 | void __iomem *base = data->base; | |
2849dd8b | 599 | |
55538fbc | 600 | dev_pm_opp_remove_all_dynamic(cpu_dev); |
51c843cf | 601 | dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); |
275157b3 | 602 | qcom_cpufreq_hw_lmh_exit(data); |
2849dd8b | 603 | kfree(policy->freq_table); |
67fc209b SG |
604 | kfree(data); |
605 | iounmap(base); | |
606 | release_mem_region(res->start, resource_size(res)); | |
2849dd8b TD |
607 | |
608 | return 0; | |
609 | } | |
610 | ||
ef8ee1cb BA |
611 | static void qcom_cpufreq_ready(struct cpufreq_policy *policy) |
612 | { | |
613 | struct qcom_cpufreq_data *data = policy->driver_data; | |
614 | ||
615 | if (data->throttle_irq >= 0) | |
616 | enable_irq(data->throttle_irq); | |
617 | } | |
618 | ||
2849dd8b TD |
619 | static struct freq_attr *qcom_cpufreq_hw_attr[] = { |
620 | &cpufreq_freq_attr_scaling_available_freqs, | |
621 | &cpufreq_freq_attr_scaling_boost_freqs, | |
622 | NULL | |
623 | }; | |
624 | ||
625 | static struct cpufreq_driver cpufreq_qcom_hw_driver = { | |
5ae4a4b4 | 626 | .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | |
4c5ff1c8 AK |
627 | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | |
628 | CPUFREQ_IS_COOLING_DEV, | |
2849dd8b TD |
629 | .verify = cpufreq_generic_frequency_table_verify, |
630 | .target_index = qcom_cpufreq_hw_target_index, | |
631 | .get = qcom_cpufreq_hw_get, | |
632 | .init = qcom_cpufreq_hw_cpu_init, | |
633 | .exit = qcom_cpufreq_hw_cpu_exit, | |
a1eb080a DB |
634 | .online = qcom_cpufreq_hw_cpu_online, |
635 | .offline = qcom_cpufreq_hw_cpu_offline, | |
e96c2153 | 636 | .register_em = cpufreq_register_em_with_opp, |
2849dd8b TD |
637 | .fast_switch = qcom_cpufreq_hw_fast_switch, |
638 | .name = "qcom-cpufreq-hw", | |
639 | .attr = qcom_cpufreq_hw_attr, | |
ef8ee1cb | 640 | .ready = qcom_cpufreq_ready, |
2849dd8b TD |
641 | }; |
642 | ||
643 | static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) | |
644 | { | |
51c843cf | 645 | struct device *cpu_dev; |
2849dd8b TD |
646 | struct clk *clk; |
647 | int ret; | |
648 | ||
649 | clk = clk_get(&pdev->dev, "xo"); | |
650 | if (IS_ERR(clk)) | |
651 | return PTR_ERR(clk); | |
652 | ||
653 | xo_rate = clk_get_rate(clk); | |
654 | clk_put(clk); | |
655 | ||
656 | clk = clk_get(&pdev->dev, "alternate"); | |
657 | if (IS_ERR(clk)) | |
658 | return PTR_ERR(clk); | |
659 | ||
660 | cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; | |
661 | clk_put(clk); | |
662 | ||
bd74e286 | 663 | cpufreq_qcom_hw_driver.driver_data = pdev; |
2849dd8b | 664 | |
51c843cf SS |
665 | /* Check for optional interconnect paths on CPU0 */ |
666 | cpu_dev = get_cpu_device(0); | |
667 | if (!cpu_dev) | |
668 | return -EPROBE_DEFER; | |
669 | ||
670 | ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); | |
671 | if (ret) | |
672 | return ret; | |
673 | ||
2849dd8b TD |
674 | ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver); |
675 | if (ret) | |
676 | dev_err(&pdev->dev, "CPUFreq HW driver failed to register\n"); | |
677 | else | |
678 | dev_dbg(&pdev->dev, "QCOM CPUFreq HW driver initialized\n"); | |
679 | ||
680 | return ret; | |
681 | } | |
682 | ||
683 | static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) | |
684 | { | |
685 | return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); | |
686 | } | |
687 | ||
2849dd8b TD |
688 | static struct platform_driver qcom_cpufreq_hw_driver = { |
689 | .probe = qcom_cpufreq_hw_driver_probe, | |
690 | .remove = qcom_cpufreq_hw_driver_remove, | |
691 | .driver = { | |
692 | .name = "qcom-cpufreq-hw", | |
693 | .of_match_table = qcom_cpufreq_hw_match, | |
694 | }, | |
695 | }; | |
696 | ||
697 | static int __init qcom_cpufreq_hw_init(void) | |
698 | { | |
699 | return platform_driver_register(&qcom_cpufreq_hw_driver); | |
700 | } | |
11ff4bdd | 701 | postcore_initcall(qcom_cpufreq_hw_init); |
2849dd8b TD |
702 | |
703 | static void __exit qcom_cpufreq_hw_exit(void) | |
704 | { | |
705 | platform_driver_unregister(&qcom_cpufreq_hw_driver); | |
706 | } | |
707 | module_exit(qcom_cpufreq_hw_exit); | |
708 | ||
709 | MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver"); | |
710 | MODULE_LICENSE("GPL v2"); |