Commit | Line | Data |
---|---|---|
2849dd8b TD |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2018, The Linux Foundation. All rights reserved. | |
4 | */ | |
5 | ||
6 | #include <linux/bitfield.h> | |
7 | #include <linux/cpufreq.h> | |
8 | #include <linux/init.h> | |
51c843cf | 9 | #include <linux/interconnect.h> |
275157b3 | 10 | #include <linux/interrupt.h> |
2849dd8b TD |
11 | #include <linux/kernel.h> |
12 | #include <linux/module.h> | |
13 | #include <linux/of_address.h> | |
14 | #include <linux/of_platform.h> | |
55538fbc | 15 | #include <linux/pm_opp.h> |
2849dd8b | 16 | #include <linux/slab.h> |
275157b3 | 17 | #include <linux/spinlock.h> |
2849dd8b TD |
18 | |
19 | #define LUT_MAX_ENTRIES 40U | |
20 | #define LUT_SRC GENMASK(31, 30) | |
21 | #define LUT_L_VAL GENMASK(7, 0) | |
22 | #define LUT_CORE_COUNT GENMASK(18, 16) | |
55538fbc | 23 | #define LUT_VOLT GENMASK(11, 0) |
2849dd8b | 24 | #define CLK_HW_DIV 2 |
0eae1e37 | 25 | #define LUT_TURBO_IND 1 |
2849dd8b | 26 | |
275157b3 TG |
27 | #define HZ_PER_KHZ 1000 |
28 | ||
dcd1fd72 MS |
29 | struct qcom_cpufreq_soc_data { |
30 | u32 reg_enable; | |
c377d4ba | 31 | u32 reg_dcvs_ctrl; |
dcd1fd72 MS |
32 | u32 reg_freq_lut; |
33 | u32 reg_volt_lut; | |
275157b3 | 34 | u32 reg_current_vote; |
dcd1fd72 MS |
35 | u32 reg_perf_state; |
36 | u8 lut_row_size; | |
37 | }; | |
38 | ||
39 | struct qcom_cpufreq_data { | |
40 | void __iomem *base; | |
67fc209b | 41 | struct resource *res; |
dcd1fd72 | 42 | const struct qcom_cpufreq_soc_data *soc_data; |
275157b3 TG |
43 | |
44 | /* | |
45 | * Mutex to synchronize between de-init sequence and re-starting LMh | |
46 | * polling/interrupts | |
47 | */ | |
48 | struct mutex throttle_lock; | |
49 | int throttle_irq; | |
be6592ed | 50 | char irq_name[15]; |
275157b3 TG |
51 | bool cancel_throttle; |
52 | struct delayed_work throttle_work; | |
53 | struct cpufreq_policy *policy; | |
c377d4ba BA |
54 | |
55 | bool per_core_dcvs; | |
dcd1fd72 | 56 | }; |
2849dd8b TD |
57 | |
58 | static unsigned long cpu_hw_rate, xo_rate; | |
51c843cf SS |
59 | static bool icc_scaling_enabled; |
60 | ||
61 | static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, | |
62 | unsigned long freq_khz) | |
63 | { | |
64 | unsigned long freq_hz = freq_khz * 1000; | |
65 | struct dev_pm_opp *opp; | |
66 | struct device *dev; | |
67 | int ret; | |
68 | ||
69 | dev = get_cpu_device(policy->cpu); | |
70 | if (!dev) | |
71 | return -ENODEV; | |
72 | ||
73 | opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true); | |
74 | if (IS_ERR(opp)) | |
75 | return PTR_ERR(opp); | |
76 | ||
8d25157f | 77 | ret = dev_pm_opp_set_opp(dev, opp); |
51c843cf SS |
78 | dev_pm_opp_put(opp); |
79 | return ret; | |
80 | } | |
81 | ||
82 | static int qcom_cpufreq_update_opp(struct device *cpu_dev, | |
83 | unsigned long freq_khz, | |
84 | unsigned long volt) | |
85 | { | |
86 | unsigned long freq_hz = freq_khz * 1000; | |
87 | int ret; | |
88 | ||
89 | /* Skip voltage update if the opp table is not available */ | |
90 | if (!icc_scaling_enabled) | |
91 | return dev_pm_opp_add(cpu_dev, freq_hz, volt); | |
92 | ||
93 | ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt); | |
94 | if (ret) { | |
95 | dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz); | |
96 | return ret; | |
97 | } | |
98 | ||
99 | return dev_pm_opp_enable(cpu_dev, freq_hz); | |
100 | } | |
2849dd8b TD |
101 | |
102 | static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, | |
103 | unsigned int index) | |
104 | { | |
dcd1fd72 MS |
105 | struct qcom_cpufreq_data *data = policy->driver_data; |
106 | const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; | |
ada54f35 | 107 | unsigned long freq = policy->freq_table[index].frequency; |
c377d4ba | 108 | unsigned int i; |
2849dd8b | 109 | |
dcd1fd72 | 110 | writel_relaxed(index, data->base + soc_data->reg_perf_state); |
2849dd8b | 111 | |
c377d4ba BA |
112 | if (data->per_core_dcvs) |
113 | for (i = 1; i < cpumask_weight(policy->related_cpus); i++) | |
114 | writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); | |
115 | ||
51c843cf SS |
116 | if (icc_scaling_enabled) |
117 | qcom_cpufreq_set_bw(policy, freq); | |
118 | ||
2849dd8b TD |
119 | return 0; |
120 | } | |
121 | ||
122 | static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) | |
123 | { | |
dcd1fd72 MS |
124 | struct qcom_cpufreq_data *data; |
125 | const struct qcom_cpufreq_soc_data *soc_data; | |
2849dd8b TD |
126 | struct cpufreq_policy *policy; |
127 | unsigned int index; | |
128 | ||
129 | policy = cpufreq_cpu_get_raw(cpu); | |
130 | if (!policy) | |
131 | return 0; | |
132 | ||
dcd1fd72 MS |
133 | data = policy->driver_data; |
134 | soc_data = data->soc_data; | |
2849dd8b | 135 | |
dcd1fd72 | 136 | index = readl_relaxed(data->base + soc_data->reg_perf_state); |
2849dd8b TD |
137 | index = min(index, LUT_MAX_ENTRIES - 1); |
138 | ||
139 | return policy->freq_table[index].frequency; | |
140 | } | |
141 | ||
142 | static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, | |
143 | unsigned int target_freq) | |
144 | { | |
dcd1fd72 MS |
145 | struct qcom_cpufreq_data *data = policy->driver_data; |
146 | const struct qcom_cpufreq_soc_data *soc_data = data->soc_data; | |
292072c3 | 147 | unsigned int index; |
c377d4ba | 148 | unsigned int i; |
2849dd8b TD |
149 | |
150 | index = policy->cached_resolved_idx; | |
dcd1fd72 | 151 | writel_relaxed(index, data->base + soc_data->reg_perf_state); |
2849dd8b | 152 | |
c377d4ba BA |
153 | if (data->per_core_dcvs) |
154 | for (i = 1; i < cpumask_weight(policy->related_cpus); i++) | |
155 | writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); | |
156 | ||
1a0419b0 | 157 | return policy->freq_table[index].frequency; |
2849dd8b TD |
158 | } |
159 | ||
55538fbc | 160 | static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, |
dcd1fd72 | 161 | struct cpufreq_policy *policy) |
2849dd8b | 162 | { |
0eae1e37 | 163 | u32 data, src, lval, i, core_count, prev_freq = 0, freq; |
55538fbc | 164 | u32 volt; |
2849dd8b | 165 | struct cpufreq_frequency_table *table; |
51c843cf SS |
166 | struct dev_pm_opp *opp; |
167 | unsigned long rate; | |
168 | int ret; | |
dcd1fd72 MS |
169 | struct qcom_cpufreq_data *drv_data = policy->driver_data; |
170 | const struct qcom_cpufreq_soc_data *soc_data = drv_data->soc_data; | |
2849dd8b TD |
171 | |
172 | table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); | |
173 | if (!table) | |
174 | return -ENOMEM; | |
175 | ||
51c843cf SS |
176 | ret = dev_pm_opp_of_add_table(cpu_dev); |
177 | if (!ret) { | |
178 | /* Disable all opps and cross-validate against LUT later */ | |
179 | icc_scaling_enabled = true; | |
180 | for (rate = 0; ; rate++) { | |
181 | opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); | |
182 | if (IS_ERR(opp)) | |
183 | break; | |
184 | ||
185 | dev_pm_opp_put(opp); | |
186 | dev_pm_opp_disable(cpu_dev, rate); | |
187 | } | |
188 | } else if (ret != -ENODEV) { | |
189 | dev_err(cpu_dev, "Invalid opp table in device tree\n"); | |
190 | return ret; | |
191 | } else { | |
afdb219b | 192 | policy->fast_switch_possible = true; |
51c843cf SS |
193 | icc_scaling_enabled = false; |
194 | } | |
195 | ||
2849dd8b | 196 | for (i = 0; i < LUT_MAX_ENTRIES; i++) { |
dcd1fd72 MS |
197 | data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + |
198 | i * soc_data->lut_row_size); | |
2849dd8b TD |
199 | src = FIELD_GET(LUT_SRC, data); |
200 | lval = FIELD_GET(LUT_L_VAL, data); | |
201 | core_count = FIELD_GET(LUT_CORE_COUNT, data); | |
202 | ||
dcd1fd72 MS |
203 | data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + |
204 | i * soc_data->lut_row_size); | |
55538fbc TD |
205 | volt = FIELD_GET(LUT_VOLT, data) * 1000; |
206 | ||
2849dd8b TD |
207 | if (src) |
208 | freq = xo_rate * lval / 1000; | |
209 | else | |
210 | freq = cpu_hw_rate / 1000; | |
211 | ||
0eae1e37 | 212 | if (freq != prev_freq && core_count != LUT_TURBO_IND) { |
bc9b9c5a MK |
213 | if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { |
214 | table[i].frequency = freq; | |
215 | dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, | |
2849dd8b | 216 | freq, core_count); |
bc9b9c5a MK |
217 | } else { |
218 | dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); | |
219 | table[i].frequency = CPUFREQ_ENTRY_INVALID; | |
220 | } | |
221 | ||
0eae1e37 | 222 | } else if (core_count == LUT_TURBO_IND) { |
55538fbc | 223 | table[i].frequency = CPUFREQ_ENTRY_INVALID; |
2849dd8b TD |
224 | } |
225 | ||
226 | /* | |
227 | * Two of the same frequencies with the same core counts means | |
228 | * end of table | |
229 | */ | |
0eae1e37 | 230 | if (i > 0 && prev_freq == freq) { |
2849dd8b TD |
231 | struct cpufreq_frequency_table *prev = &table[i - 1]; |
232 | ||
233 | /* | |
234 | * Only treat the last frequency that might be a boost | |
235 | * as the boost frequency | |
236 | */ | |
0eae1e37 | 237 | if (prev->frequency == CPUFREQ_ENTRY_INVALID) { |
bc9b9c5a MK |
238 | if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { |
239 | prev->frequency = prev_freq; | |
240 | prev->flags = CPUFREQ_BOOST_FREQ; | |
241 | } else { | |
242 | dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", | |
243 | freq); | |
244 | } | |
2849dd8b TD |
245 | } |
246 | ||
247 | break; | |
248 | } | |
249 | ||
2849dd8b TD |
250 | prev_freq = freq; |
251 | } | |
252 | ||
253 | table[i].frequency = CPUFREQ_TABLE_END; | |
254 | policy->freq_table = table; | |
55538fbc | 255 | dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); |
2849dd8b TD |
256 | |
257 | return 0; | |
258 | } | |
259 | ||
260 | static void qcom_get_related_cpus(int index, struct cpumask *m) | |
261 | { | |
262 | struct device_node *cpu_np; | |
263 | struct of_phandle_args args; | |
264 | int cpu, ret; | |
265 | ||
266 | for_each_possible_cpu(cpu) { | |
267 | cpu_np = of_cpu_device_node_get(cpu); | |
268 | if (!cpu_np) | |
269 | continue; | |
270 | ||
271 | ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", | |
272 | "#freq-domain-cells", 0, | |
273 | &args); | |
274 | of_node_put(cpu_np); | |
275 | if (ret < 0) | |
276 | continue; | |
277 | ||
278 | if (index == args.args[0]) | |
279 | cpumask_set_cpu(cpu, m); | |
280 | } | |
281 | } | |
282 | ||
275157b3 TG |
283 | static unsigned int qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data) |
284 | { | |
285 | unsigned int val = readl_relaxed(data->base + data->soc_data->reg_current_vote); | |
286 | ||
287 | return (val & 0x3FF) * 19200; | |
288 | } | |
289 | ||
290 | static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) | |
291 | { | |
275157b3 TG |
292 | struct cpufreq_policy *policy = data->policy; |
293 | int cpu = cpumask_first(policy->cpus); | |
294 | struct device *dev = get_cpu_device(cpu); | |
0258cb19 | 295 | unsigned long freq_hz, throttled_freq; |
275157b3 TG |
296 | struct dev_pm_opp *opp; |
297 | unsigned int freq; | |
298 | ||
299 | /* | |
300 | * Get the h/w throttled frequency, normalize it using the | |
301 | * registered opp table and use it to calculate thermal pressure. | |
302 | */ | |
303 | freq = qcom_lmh_get_throttle_freq(data); | |
304 | freq_hz = freq * HZ_PER_KHZ; | |
305 | ||
306 | opp = dev_pm_opp_find_freq_floor(dev, &freq_hz); | |
307 | if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) | |
308 | dev_pm_opp_find_freq_ceil(dev, &freq_hz); | |
309 | ||
310 | throttled_freq = freq_hz / HZ_PER_KHZ; | |
311 | ||
0258cb19 LL |
312 | /* Update thermal pressure (the boost frequencies are accepted) */ |
313 | arch_update_thermal_pressure(policy->related_cpus, throttled_freq); | |
275157b3 TG |
314 | |
315 | /* | |
316 | * In the unlikely case policy is unregistered do not enable | |
317 | * polling or h/w interrupt | |
318 | */ | |
319 | mutex_lock(&data->throttle_lock); | |
320 | if (data->cancel_throttle) | |
321 | goto out; | |
322 | ||
323 | /* | |
324 | * If h/w throttled frequency is higher than what cpufreq has requested | |
325 | * for, then stop polling and switch back to interrupt mechanism. | |
326 | */ | |
327 | if (throttled_freq >= qcom_cpufreq_hw_get(cpu)) | |
328 | enable_irq(data->throttle_irq); | |
329 | else | |
330 | mod_delayed_work(system_highpri_wq, &data->throttle_work, | |
331 | msecs_to_jiffies(10)); | |
332 | ||
333 | out: | |
334 | mutex_unlock(&data->throttle_lock); | |
335 | } | |
336 | ||
337 | static void qcom_lmh_dcvs_poll(struct work_struct *work) | |
338 | { | |
339 | struct qcom_cpufreq_data *data; | |
340 | ||
341 | data = container_of(work, struct qcom_cpufreq_data, throttle_work.work); | |
342 | qcom_lmh_dcvs_notify(data); | |
343 | } | |
344 | ||
345 | static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data) | |
346 | { | |
347 | struct qcom_cpufreq_data *c_data = data; | |
348 | ||
349 | /* Disable interrupt and enable polling */ | |
350 | disable_irq_nosync(c_data->throttle_irq); | |
e0e27c3d | 351 | schedule_delayed_work(&c_data->throttle_work, 0); |
275157b3 | 352 | |
e0e27c3d | 353 | return IRQ_HANDLED; |
275157b3 TG |
354 | } |
355 | ||
dcd1fd72 MS |
356 | static const struct qcom_cpufreq_soc_data qcom_soc_data = { |
357 | .reg_enable = 0x0, | |
c377d4ba | 358 | .reg_dcvs_ctrl = 0xbc, |
dcd1fd72 MS |
359 | .reg_freq_lut = 0x110, |
360 | .reg_volt_lut = 0x114, | |
275157b3 | 361 | .reg_current_vote = 0x704, |
dcd1fd72 MS |
362 | .reg_perf_state = 0x920, |
363 | .lut_row_size = 32, | |
364 | }; | |
365 | ||
49b59f4c MS |
366 | static const struct qcom_cpufreq_soc_data epss_soc_data = { |
367 | .reg_enable = 0x0, | |
c377d4ba | 368 | .reg_dcvs_ctrl = 0xb0, |
49b59f4c MS |
369 | .reg_freq_lut = 0x100, |
370 | .reg_volt_lut = 0x200, | |
371 | .reg_perf_state = 0x320, | |
372 | .lut_row_size = 4, | |
373 | }; | |
374 | ||
dcd1fd72 MS |
375 | static const struct of_device_id qcom_cpufreq_hw_match[] = { |
376 | { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, | |
49b59f4c | 377 | { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, |
dcd1fd72 MS |
378 | {} |
379 | }; | |
380 | MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); | |
381 | ||
275157b3 TG |
382 | static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) |
383 | { | |
384 | struct qcom_cpufreq_data *data = policy->driver_data; | |
385 | struct platform_device *pdev = cpufreq_get_driver_data(); | |
275157b3 TG |
386 | int ret; |
387 | ||
388 | /* | |
389 | * Look for LMh interrupt. If no interrupt line is specified / | |
390 | * if there is an error, allow cpufreq to be enabled as usual. | |
391 | */ | |
8f5783ad SB |
392 | data->throttle_irq = platform_get_irq_optional(pdev, index); |
393 | if (data->throttle_irq == -ENXIO) | |
394 | return 0; | |
395 | if (data->throttle_irq < 0) | |
396 | return data->throttle_irq; | |
275157b3 TG |
397 | |
398 | data->cancel_throttle = false; | |
399 | data->policy = policy; | |
400 | ||
401 | mutex_init(&data->throttle_lock); | |
402 | INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll); | |
403 | ||
be6592ed | 404 | snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu); |
275157b3 | 405 | ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq, |
ef8ee1cb | 406 | IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data); |
275157b3 | 407 | if (ret) { |
be6592ed | 408 | dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret); |
275157b3 TG |
409 | return 0; |
410 | } | |
411 | ||
3ed6dfbd VZ |
412 | ret = irq_set_affinity_hint(data->throttle_irq, policy->cpus); |
413 | if (ret) | |
414 | dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", | |
415 | data->irq_name, data->throttle_irq); | |
416 | ||
275157b3 TG |
417 | return 0; |
418 | } | |
419 | ||
420 | static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) | |
421 | { | |
422 | if (data->throttle_irq <= 0) | |
423 | return; | |
424 | ||
425 | mutex_lock(&data->throttle_lock); | |
426 | data->cancel_throttle = true; | |
427 | mutex_unlock(&data->throttle_lock); | |
428 | ||
429 | cancel_delayed_work_sync(&data->throttle_work); | |
430 | free_irq(data->throttle_irq, data); | |
431 | } | |
432 | ||
2849dd8b TD |
433 | static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) |
434 | { | |
bd74e286 MS |
435 | struct platform_device *pdev = cpufreq_get_driver_data(); |
436 | struct device *dev = &pdev->dev; | |
2849dd8b TD |
437 | struct of_phandle_args args; |
438 | struct device_node *cpu_np; | |
55538fbc | 439 | struct device *cpu_dev; |
67fc209b | 440 | struct resource *res; |
2849dd8b | 441 | void __iomem *base; |
dcd1fd72 | 442 | struct qcom_cpufreq_data *data; |
2849dd8b TD |
443 | int ret, index; |
444 | ||
55538fbc TD |
445 | cpu_dev = get_cpu_device(policy->cpu); |
446 | if (!cpu_dev) { | |
447 | pr_err("%s: failed to get cpu%d device\n", __func__, | |
448 | policy->cpu); | |
449 | return -ENODEV; | |
450 | } | |
451 | ||
2849dd8b TD |
452 | cpu_np = of_cpu_device_node_get(policy->cpu); |
453 | if (!cpu_np) | |
454 | return -EINVAL; | |
455 | ||
456 | ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", | |
457 | "#freq-domain-cells", 0, &args); | |
458 | of_node_put(cpu_np); | |
459 | if (ret) | |
460 | return ret; | |
461 | ||
462 | index = args.args[0]; | |
463 | ||
67fc209b SG |
464 | res = platform_get_resource(pdev, IORESOURCE_MEM, index); |
465 | if (!res) { | |
466 | dev_err(dev, "failed to get mem resource %d\n", index); | |
467 | return -ENODEV; | |
468 | } | |
469 | ||
470 | if (!request_mem_region(res->start, resource_size(res), res->name)) { | |
471 | dev_err(dev, "failed to request resource %pR\n", res); | |
472 | return -EBUSY; | |
473 | } | |
2849dd8b | 474 | |
67fc209b | 475 | base = ioremap(res->start, resource_size(res)); |
536eb97a | 476 | if (!base) { |
67fc209b | 477 | dev_err(dev, "failed to map resource %pR\n", res); |
536eb97a | 478 | ret = -ENOMEM; |
67fc209b SG |
479 | goto release_region; |
480 | } | |
481 | ||
482 | data = kzalloc(sizeof(*data), GFP_KERNEL); | |
dcd1fd72 MS |
483 | if (!data) { |
484 | ret = -ENOMEM; | |
67fc209b | 485 | goto unmap_base; |
dcd1fd72 MS |
486 | } |
487 | ||
488 | data->soc_data = of_device_get_match_data(&pdev->dev); | |
489 | data->base = base; | |
67fc209b | 490 | data->res = res; |
2849dd8b TD |
491 | |
492 | /* HW should be in enabled state to proceed */ | |
dcd1fd72 | 493 | if (!(readl_relaxed(base + data->soc_data->reg_enable) & 0x1)) { |
2849dd8b TD |
494 | dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); |
495 | ret = -ENODEV; | |
496 | goto error; | |
497 | } | |
498 | ||
c377d4ba BA |
499 | if (readl_relaxed(base + data->soc_data->reg_dcvs_ctrl) & 0x1) |
500 | data->per_core_dcvs = true; | |
501 | ||
2849dd8b | 502 | qcom_get_related_cpus(index, policy->cpus); |
b48cd0d1 | 503 | if (cpumask_empty(policy->cpus)) { |
2849dd8b TD |
504 | dev_err(dev, "Domain-%d failed to get related CPUs\n", index); |
505 | ret = -ENOENT; | |
506 | goto error; | |
507 | } | |
508 | ||
dcd1fd72 | 509 | policy->driver_data = data; |
f0712ace | 510 | policy->dvfs_possible_from_any_cpu = true; |
2849dd8b | 511 | |
dcd1fd72 | 512 | ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); |
2849dd8b TD |
513 | if (ret) { |
514 | dev_err(dev, "Domain-%d failed to read LUT\n", index); | |
515 | goto error; | |
516 | } | |
517 | ||
55538fbc TD |
518 | ret = dev_pm_opp_get_opp_count(cpu_dev); |
519 | if (ret <= 0) { | |
520 | dev_err(cpu_dev, "Failed to add OPPs\n"); | |
521 | ret = -ENODEV; | |
522 | goto error; | |
523 | } | |
524 | ||
26699172 SG |
525 | if (policy_has_boost_freq(policy)) { |
526 | ret = cpufreq_enable_boost_support(); | |
527 | if (ret) | |
528 | dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); | |
529 | } | |
530 | ||
275157b3 TG |
531 | ret = qcom_cpufreq_hw_lmh_init(policy, index); |
532 | if (ret) | |
533 | goto error; | |
534 | ||
2849dd8b TD |
535 | return 0; |
536 | error: | |
67fc209b SG |
537 | kfree(data); |
538 | unmap_base: | |
02fc4095 | 539 | iounmap(base); |
67fc209b SG |
540 | release_region: |
541 | release_mem_region(res->start, resource_size(res)); | |
2849dd8b TD |
542 | return ret; |
543 | } | |
544 | ||
545 | static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) | |
546 | { | |
55538fbc | 547 | struct device *cpu_dev = get_cpu_device(policy->cpu); |
dcd1fd72 | 548 | struct qcom_cpufreq_data *data = policy->driver_data; |
67fc209b SG |
549 | struct resource *res = data->res; |
550 | void __iomem *base = data->base; | |
2849dd8b | 551 | |
55538fbc | 552 | dev_pm_opp_remove_all_dynamic(cpu_dev); |
51c843cf | 553 | dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); |
275157b3 | 554 | qcom_cpufreq_hw_lmh_exit(data); |
2849dd8b | 555 | kfree(policy->freq_table); |
67fc209b SG |
556 | kfree(data); |
557 | iounmap(base); | |
558 | release_mem_region(res->start, resource_size(res)); | |
2849dd8b TD |
559 | |
560 | return 0; | |
561 | } | |
562 | ||
ef8ee1cb BA |
563 | static void qcom_cpufreq_ready(struct cpufreq_policy *policy) |
564 | { | |
565 | struct qcom_cpufreq_data *data = policy->driver_data; | |
566 | ||
567 | if (data->throttle_irq >= 0) | |
568 | enable_irq(data->throttle_irq); | |
569 | } | |
570 | ||
2849dd8b TD |
571 | static struct freq_attr *qcom_cpufreq_hw_attr[] = { |
572 | &cpufreq_freq_attr_scaling_available_freqs, | |
573 | &cpufreq_freq_attr_scaling_boost_freqs, | |
574 | NULL | |
575 | }; | |
576 | ||
577 | static struct cpufreq_driver cpufreq_qcom_hw_driver = { | |
5ae4a4b4 | 578 | .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | |
4c5ff1c8 AK |
579 | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | |
580 | CPUFREQ_IS_COOLING_DEV, | |
2849dd8b TD |
581 | .verify = cpufreq_generic_frequency_table_verify, |
582 | .target_index = qcom_cpufreq_hw_target_index, | |
583 | .get = qcom_cpufreq_hw_get, | |
584 | .init = qcom_cpufreq_hw_cpu_init, | |
585 | .exit = qcom_cpufreq_hw_cpu_exit, | |
e96c2153 | 586 | .register_em = cpufreq_register_em_with_opp, |
2849dd8b TD |
587 | .fast_switch = qcom_cpufreq_hw_fast_switch, |
588 | .name = "qcom-cpufreq-hw", | |
589 | .attr = qcom_cpufreq_hw_attr, | |
ef8ee1cb | 590 | .ready = qcom_cpufreq_ready, |
2849dd8b TD |
591 | }; |
592 | ||
593 | static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) | |
594 | { | |
51c843cf | 595 | struct device *cpu_dev; |
2849dd8b TD |
596 | struct clk *clk; |
597 | int ret; | |
598 | ||
599 | clk = clk_get(&pdev->dev, "xo"); | |
600 | if (IS_ERR(clk)) | |
601 | return PTR_ERR(clk); | |
602 | ||
603 | xo_rate = clk_get_rate(clk); | |
604 | clk_put(clk); | |
605 | ||
606 | clk = clk_get(&pdev->dev, "alternate"); | |
607 | if (IS_ERR(clk)) | |
608 | return PTR_ERR(clk); | |
609 | ||
610 | cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; | |
611 | clk_put(clk); | |
612 | ||
bd74e286 | 613 | cpufreq_qcom_hw_driver.driver_data = pdev; |
2849dd8b | 614 | |
51c843cf SS |
615 | /* Check for optional interconnect paths on CPU0 */ |
616 | cpu_dev = get_cpu_device(0); | |
617 | if (!cpu_dev) | |
618 | return -EPROBE_DEFER; | |
619 | ||
620 | ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); | |
621 | if (ret) | |
622 | return ret; | |
623 | ||
2849dd8b TD |
624 | ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver); |
625 | if (ret) | |
626 | dev_err(&pdev->dev, "CPUFreq HW driver failed to register\n"); | |
627 | else | |
628 | dev_dbg(&pdev->dev, "QCOM CPUFreq HW driver initialized\n"); | |
629 | ||
630 | return ret; | |
631 | } | |
632 | ||
633 | static int qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) | |
634 | { | |
635 | return cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); | |
636 | } | |
637 | ||
2849dd8b TD |
638 | static struct platform_driver qcom_cpufreq_hw_driver = { |
639 | .probe = qcom_cpufreq_hw_driver_probe, | |
640 | .remove = qcom_cpufreq_hw_driver_remove, | |
641 | .driver = { | |
642 | .name = "qcom-cpufreq-hw", | |
643 | .of_match_table = qcom_cpufreq_hw_match, | |
644 | }, | |
645 | }; | |
646 | ||
647 | static int __init qcom_cpufreq_hw_init(void) | |
648 | { | |
649 | return platform_driver_register(&qcom_cpufreq_hw_driver); | |
650 | } | |
11ff4bdd | 651 | postcore_initcall(qcom_cpufreq_hw_init); |
2849dd8b TD |
652 | |
653 | static void __exit qcom_cpufreq_hw_exit(void) | |
654 | { | |
655 | platform_driver_unregister(&qcom_cpufreq_hw_driver); | |
656 | } | |
657 | module_exit(qcom_cpufreq_hw_exit); | |
658 | ||
659 | MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver"); | |
660 | MODULE_LICENSE("GPL v2"); |