Commit | Line | Data |
---|---|---|
2849dd8b TD |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2018, The Linux Foundation. All rights reserved. | |
4 | */ | |
5 | ||
6 | #include <linux/bitfield.h> | |
4370232c | 7 | #include <linux/clk-provider.h> |
2849dd8b TD |
8 | #include <linux/cpufreq.h> |
9 | #include <linux/init.h> | |
51c843cf | 10 | #include <linux/interconnect.h> |
275157b3 | 11 | #include <linux/interrupt.h> |
2849dd8b TD |
12 | #include <linux/kernel.h> |
13 | #include <linux/module.h> | |
21bb32b1 RH |
14 | #include <linux/of.h> |
15 | #include <linux/platform_device.h> | |
55538fbc | 16 | #include <linux/pm_opp.h> |
2849dd8b | 17 | #include <linux/slab.h> |
275157b3 | 18 | #include <linux/spinlock.h> |
9de0d75b | 19 | #include <linux/units.h> |
2849dd8b TD |
20 | |
21 | #define LUT_MAX_ENTRIES 40U | |
22 | #define LUT_SRC GENMASK(31, 30) | |
23 | #define LUT_L_VAL GENMASK(7, 0) | |
24 | #define LUT_CORE_COUNT GENMASK(18, 16) | |
55538fbc | 25 | #define LUT_VOLT GENMASK(11, 0) |
2849dd8b | 26 | #define CLK_HW_DIV 2 |
0eae1e37 | 27 | #define LUT_TURBO_IND 1 |
2849dd8b | 28 | |
e4e64486 VZ |
29 | #define GT_IRQ_STATUS BIT(2) |
30 | ||
5f19d096 | 31 | #define MAX_FREQ_DOMAINS 4 |
35527c67 | 32 | |
dcd1fd72 MS |
33 | struct qcom_cpufreq_soc_data { |
34 | u32 reg_enable; | |
f84ccad5 | 35 | u32 reg_domain_state; |
c377d4ba | 36 | u32 reg_dcvs_ctrl; |
dcd1fd72 MS |
37 | u32 reg_freq_lut; |
38 | u32 reg_volt_lut; | |
e4e64486 | 39 | u32 reg_intr_clr; |
275157b3 | 40 | u32 reg_current_vote; |
dcd1fd72 MS |
41 | u32 reg_perf_state; |
42 | u8 lut_row_size; | |
43 | }; | |
44 | ||
45 | struct qcom_cpufreq_data { | |
46 | void __iomem *base; | |
275157b3 TG |
47 | |
48 | /* | |
49 | * Mutex to synchronize between de-init sequence and re-starting LMh | |
50 | * polling/interrupts | |
51 | */ | |
52 | struct mutex throttle_lock; | |
53 | int throttle_irq; | |
be6592ed | 54 | char irq_name[15]; |
275157b3 TG |
55 | bool cancel_throttle; |
56 | struct delayed_work throttle_work; | |
57 | struct cpufreq_policy *policy; | |
4370232c | 58 | struct clk_hw cpu_clk; |
c377d4ba BA |
59 | |
60 | bool per_core_dcvs; | |
dcd1fd72 | 61 | }; |
2849dd8b | 62 | |
054a3ef6 MS |
63 | static struct { |
64 | struct qcom_cpufreq_data *data; | |
4f796170 | 65 | const struct qcom_cpufreq_soc_data *soc_data; |
054a3ef6 MS |
66 | } qcom_cpufreq; |
67 | ||
2849dd8b | 68 | static unsigned long cpu_hw_rate, xo_rate; |
51c843cf SS |
69 | static bool icc_scaling_enabled; |
70 | ||
71 | static int qcom_cpufreq_set_bw(struct cpufreq_policy *policy, | |
72 | unsigned long freq_khz) | |
73 | { | |
74 | unsigned long freq_hz = freq_khz * 1000; | |
75 | struct dev_pm_opp *opp; | |
76 | struct device *dev; | |
77 | int ret; | |
78 | ||
79 | dev = get_cpu_device(policy->cpu); | |
80 | if (!dev) | |
81 | return -ENODEV; | |
82 | ||
83 | opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true); | |
84 | if (IS_ERR(opp)) | |
85 | return PTR_ERR(opp); | |
86 | ||
8d25157f | 87 | ret = dev_pm_opp_set_opp(dev, opp); |
51c843cf SS |
88 | dev_pm_opp_put(opp); |
89 | return ret; | |
90 | } | |
91 | ||
92 | static int qcom_cpufreq_update_opp(struct device *cpu_dev, | |
93 | unsigned long freq_khz, | |
94 | unsigned long volt) | |
95 | { | |
96 | unsigned long freq_hz = freq_khz * 1000; | |
97 | int ret; | |
98 | ||
99 | /* Skip voltage update if the opp table is not available */ | |
100 | if (!icc_scaling_enabled) | |
101 | return dev_pm_opp_add(cpu_dev, freq_hz, volt); | |
102 | ||
103 | ret = dev_pm_opp_adjust_voltage(cpu_dev, freq_hz, volt, volt, volt); | |
104 | if (ret) { | |
105 | dev_err(cpu_dev, "Voltage update failed freq=%ld\n", freq_khz); | |
106 | return ret; | |
107 | } | |
108 | ||
109 | return dev_pm_opp_enable(cpu_dev, freq_hz); | |
110 | } | |
2849dd8b TD |
111 | |
112 | static int qcom_cpufreq_hw_target_index(struct cpufreq_policy *policy, | |
113 | unsigned int index) | |
114 | { | |
dcd1fd72 | 115 | struct qcom_cpufreq_data *data = policy->driver_data; |
4f796170 | 116 | const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data; |
ada54f35 | 117 | unsigned long freq = policy->freq_table[index].frequency; |
c377d4ba | 118 | unsigned int i; |
2849dd8b | 119 | |
dcd1fd72 | 120 | writel_relaxed(index, data->base + soc_data->reg_perf_state); |
2849dd8b | 121 | |
c377d4ba BA |
122 | if (data->per_core_dcvs) |
123 | for (i = 1; i < cpumask_weight(policy->related_cpus); i++) | |
124 | writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); | |
125 | ||
51c843cf SS |
126 | if (icc_scaling_enabled) |
127 | qcom_cpufreq_set_bw(policy, freq); | |
128 | ||
2849dd8b TD |
129 | return 0; |
130 | } | |
131 | ||
c72cf0cb MS |
132 | static unsigned long qcom_lmh_get_throttle_freq(struct qcom_cpufreq_data *data) |
133 | { | |
134 | unsigned int lval; | |
135 | ||
136 | if (qcom_cpufreq.soc_data->reg_current_vote) | |
137 | lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_current_vote) & 0x3ff; | |
138 | else | |
139 | lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_domain_state) & 0xff; | |
140 | ||
141 | return lval * xo_rate; | |
142 | } | |
143 | ||
51be2fff DA |
144 | /* Get the frequency requested by the cpufreq core for the CPU */ |
145 | static unsigned int qcom_cpufreq_get_freq(unsigned int cpu) | |
c72cf0cb MS |
146 | { |
147 | struct qcom_cpufreq_data *data; | |
51be2fff | 148 | const struct qcom_cpufreq_soc_data *soc_data; |
c72cf0cb | 149 | struct cpufreq_policy *policy; |
51be2fff | 150 | unsigned int index; |
c72cf0cb MS |
151 | |
152 | policy = cpufreq_cpu_get_raw(cpu); | |
153 | if (!policy) | |
154 | return 0; | |
155 | ||
156 | data = policy->driver_data; | |
51be2fff | 157 | soc_data = qcom_cpufreq.soc_data; |
c72cf0cb | 158 | |
51be2fff DA |
159 | index = readl_relaxed(data->base + soc_data->reg_perf_state); |
160 | index = min(index, LUT_MAX_ENTRIES - 1); | |
161 | ||
162 | return policy->freq_table[index].frequency; | |
c72cf0cb MS |
163 | } |
164 | ||
51be2fff | 165 | static unsigned int qcom_cpufreq_hw_get(unsigned int cpu) |
2849dd8b | 166 | { |
dcd1fd72 | 167 | struct qcom_cpufreq_data *data; |
2849dd8b | 168 | struct cpufreq_policy *policy; |
2849dd8b TD |
169 | |
170 | policy = cpufreq_cpu_get_raw(cpu); | |
171 | if (!policy) | |
172 | return 0; | |
173 | ||
dcd1fd72 | 174 | data = policy->driver_data; |
2849dd8b | 175 | |
51be2fff DA |
176 | if (data->throttle_irq >= 0) |
177 | return qcom_lmh_get_throttle_freq(data) / HZ_PER_KHZ; | |
2849dd8b | 178 | |
51be2fff | 179 | return qcom_cpufreq_get_freq(cpu); |
2849dd8b TD |
180 | } |
181 | ||
182 | static unsigned int qcom_cpufreq_hw_fast_switch(struct cpufreq_policy *policy, | |
183 | unsigned int target_freq) | |
184 | { | |
dcd1fd72 | 185 | struct qcom_cpufreq_data *data = policy->driver_data; |
4f796170 | 186 | const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data; |
292072c3 | 187 | unsigned int index; |
c377d4ba | 188 | unsigned int i; |
2849dd8b TD |
189 | |
190 | index = policy->cached_resolved_idx; | |
dcd1fd72 | 191 | writel_relaxed(index, data->base + soc_data->reg_perf_state); |
2849dd8b | 192 | |
c377d4ba BA |
193 | if (data->per_core_dcvs) |
194 | for (i = 1; i < cpumask_weight(policy->related_cpus); i++) | |
195 | writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4); | |
196 | ||
1a0419b0 | 197 | return policy->freq_table[index].frequency; |
2849dd8b TD |
198 | } |
199 | ||
55538fbc | 200 | static int qcom_cpufreq_hw_read_lut(struct device *cpu_dev, |
dcd1fd72 | 201 | struct cpufreq_policy *policy) |
2849dd8b | 202 | { |
0eae1e37 | 203 | u32 data, src, lval, i, core_count, prev_freq = 0, freq; |
55538fbc | 204 | u32 volt; |
2849dd8b | 205 | struct cpufreq_frequency_table *table; |
51c843cf SS |
206 | struct dev_pm_opp *opp; |
207 | unsigned long rate; | |
208 | int ret; | |
dcd1fd72 | 209 | struct qcom_cpufreq_data *drv_data = policy->driver_data; |
4f796170 | 210 | const struct qcom_cpufreq_soc_data *soc_data = qcom_cpufreq.soc_data; |
2849dd8b TD |
211 | |
212 | table = kcalloc(LUT_MAX_ENTRIES + 1, sizeof(*table), GFP_KERNEL); | |
213 | if (!table) | |
214 | return -ENOMEM; | |
215 | ||
51c843cf SS |
216 | ret = dev_pm_opp_of_add_table(cpu_dev); |
217 | if (!ret) { | |
218 | /* Disable all opps and cross-validate against LUT later */ | |
219 | icc_scaling_enabled = true; | |
220 | for (rate = 0; ; rate++) { | |
221 | opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); | |
222 | if (IS_ERR(opp)) | |
223 | break; | |
224 | ||
225 | dev_pm_opp_put(opp); | |
226 | dev_pm_opp_disable(cpu_dev, rate); | |
227 | } | |
228 | } else if (ret != -ENODEV) { | |
229 | dev_err(cpu_dev, "Invalid opp table in device tree\n"); | |
9901c21b | 230 | kfree(table); |
51c843cf SS |
231 | return ret; |
232 | } else { | |
afdb219b | 233 | policy->fast_switch_possible = true; |
51c843cf SS |
234 | icc_scaling_enabled = false; |
235 | } | |
236 | ||
2849dd8b | 237 | for (i = 0; i < LUT_MAX_ENTRIES; i++) { |
dcd1fd72 MS |
238 | data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut + |
239 | i * soc_data->lut_row_size); | |
2849dd8b TD |
240 | src = FIELD_GET(LUT_SRC, data); |
241 | lval = FIELD_GET(LUT_L_VAL, data); | |
242 | core_count = FIELD_GET(LUT_CORE_COUNT, data); | |
243 | ||
dcd1fd72 MS |
244 | data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut + |
245 | i * soc_data->lut_row_size); | |
55538fbc TD |
246 | volt = FIELD_GET(LUT_VOLT, data) * 1000; |
247 | ||
2849dd8b TD |
248 | if (src) |
249 | freq = xo_rate * lval / 1000; | |
250 | else | |
251 | freq = cpu_hw_rate / 1000; | |
252 | ||
0eae1e37 | 253 | if (freq != prev_freq && core_count != LUT_TURBO_IND) { |
bc9b9c5a MK |
254 | if (!qcom_cpufreq_update_opp(cpu_dev, freq, volt)) { |
255 | table[i].frequency = freq; | |
256 | dev_dbg(cpu_dev, "index=%d freq=%d, core_count %d\n", i, | |
2849dd8b | 257 | freq, core_count); |
bc9b9c5a MK |
258 | } else { |
259 | dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq); | |
260 | table[i].frequency = CPUFREQ_ENTRY_INVALID; | |
261 | } | |
262 | ||
0eae1e37 | 263 | } else if (core_count == LUT_TURBO_IND) { |
55538fbc | 264 | table[i].frequency = CPUFREQ_ENTRY_INVALID; |
2849dd8b TD |
265 | } |
266 | ||
267 | /* | |
268 | * Two of the same frequencies with the same core counts means | |
269 | * end of table | |
270 | */ | |
0eae1e37 | 271 | if (i > 0 && prev_freq == freq) { |
2849dd8b TD |
272 | struct cpufreq_frequency_table *prev = &table[i - 1]; |
273 | ||
274 | /* | |
275 | * Only treat the last frequency that might be a boost | |
276 | * as the boost frequency | |
277 | */ | |
0eae1e37 | 278 | if (prev->frequency == CPUFREQ_ENTRY_INVALID) { |
bc9b9c5a MK |
279 | if (!qcom_cpufreq_update_opp(cpu_dev, prev_freq, volt)) { |
280 | prev->frequency = prev_freq; | |
281 | prev->flags = CPUFREQ_BOOST_FREQ; | |
282 | } else { | |
283 | dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", | |
284 | freq); | |
285 | } | |
2849dd8b TD |
286 | } |
287 | ||
288 | break; | |
289 | } | |
290 | ||
2849dd8b TD |
291 | prev_freq = freq; |
292 | } | |
293 | ||
294 | table[i].frequency = CPUFREQ_TABLE_END; | |
295 | policy->freq_table = table; | |
55538fbc | 296 | dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); |
2849dd8b TD |
297 | |
298 | return 0; | |
299 | } | |
300 | ||
301 | static void qcom_get_related_cpus(int index, struct cpumask *m) | |
302 | { | |
303 | struct device_node *cpu_np; | |
304 | struct of_phandle_args args; | |
305 | int cpu, ret; | |
306 | ||
307 | for_each_possible_cpu(cpu) { | |
308 | cpu_np = of_cpu_device_node_get(cpu); | |
309 | if (!cpu_np) | |
310 | continue; | |
311 | ||
312 | ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", | |
313 | "#freq-domain-cells", 0, | |
314 | &args); | |
315 | of_node_put(cpu_np); | |
316 | if (ret < 0) | |
317 | continue; | |
318 | ||
319 | if (index == args.args[0]) | |
320 | cpumask_set_cpu(cpu, m); | |
321 | } | |
322 | } | |
323 | ||
275157b3 TG |
324 | static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data) |
325 | { | |
275157b3 | 326 | struct cpufreq_policy *policy = data->policy; |
5e4f009d | 327 | int cpu = cpumask_first(policy->related_cpus); |
275157b3 | 328 | struct device *dev = get_cpu_device(cpu); |
0258cb19 | 329 | unsigned long freq_hz, throttled_freq; |
275157b3 | 330 | struct dev_pm_opp *opp; |
275157b3 TG |
331 | |
332 | /* | |
333 | * Get the h/w throttled frequency, normalize it using the | |
334 | * registered opp table and use it to calculate thermal pressure. | |
335 | */ | |
f84ccad5 | 336 | freq_hz = qcom_lmh_get_throttle_freq(data); |
275157b3 TG |
337 | |
338 | opp = dev_pm_opp_find_freq_floor(dev, &freq_hz); | |
339 | if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) | |
6240aaad | 340 | opp = dev_pm_opp_find_freq_ceil(dev, &freq_hz); |
275157b3 | 341 | |
6240aaad DB |
342 | if (IS_ERR(opp)) { |
343 | dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp); | |
344 | } else { | |
6240aaad DB |
345 | dev_pm_opp_put(opp); |
346 | } | |
275157b3 | 347 | |
91dc90fd VK |
348 | throttled_freq = freq_hz / HZ_PER_KHZ; |
349 | ||
350 | /* Update thermal pressure (the boost frequencies are accepted) */ | |
351 | arch_update_thermal_pressure(policy->related_cpus, throttled_freq); | |
352 | ||
275157b3 TG |
353 | /* |
354 | * In the unlikely case policy is unregistered do not enable | |
355 | * polling or h/w interrupt | |
356 | */ | |
357 | mutex_lock(&data->throttle_lock); | |
358 | if (data->cancel_throttle) | |
359 | goto out; | |
360 | ||
361 | /* | |
362 | * If h/w throttled frequency is higher than what cpufreq has requested | |
363 | * for, then stop polling and switch back to interrupt mechanism. | |
364 | */ | |
c72cf0cb | 365 | if (throttled_freq >= qcom_cpufreq_get_freq(cpu)) |
275157b3 TG |
366 | enable_irq(data->throttle_irq); |
367 | else | |
368 | mod_delayed_work(system_highpri_wq, &data->throttle_work, | |
369 | msecs_to_jiffies(10)); | |
370 | ||
371 | out: | |
372 | mutex_unlock(&data->throttle_lock); | |
373 | } | |
374 | ||
375 | static void qcom_lmh_dcvs_poll(struct work_struct *work) | |
376 | { | |
377 | struct qcom_cpufreq_data *data; | |
378 | ||
379 | data = container_of(work, struct qcom_cpufreq_data, throttle_work.work); | |
380 | qcom_lmh_dcvs_notify(data); | |
381 | } | |
382 | ||
383 | static irqreturn_t qcom_lmh_dcvs_handle_irq(int irq, void *data) | |
384 | { | |
385 | struct qcom_cpufreq_data *c_data = data; | |
386 | ||
387 | /* Disable interrupt and enable polling */ | |
388 | disable_irq_nosync(c_data->throttle_irq); | |
e0e27c3d | 389 | schedule_delayed_work(&c_data->throttle_work, 0); |
275157b3 | 390 | |
4f796170 | 391 | if (qcom_cpufreq.soc_data->reg_intr_clr) |
e4e64486 | 392 | writel_relaxed(GT_IRQ_STATUS, |
4f796170 | 393 | c_data->base + qcom_cpufreq.soc_data->reg_intr_clr); |
e4e64486 | 394 | |
e0e27c3d | 395 | return IRQ_HANDLED; |
275157b3 TG |
396 | } |
397 | ||
dcd1fd72 MS |
398 | static const struct qcom_cpufreq_soc_data qcom_soc_data = { |
399 | .reg_enable = 0x0, | |
c377d4ba | 400 | .reg_dcvs_ctrl = 0xbc, |
dcd1fd72 MS |
401 | .reg_freq_lut = 0x110, |
402 | .reg_volt_lut = 0x114, | |
275157b3 | 403 | .reg_current_vote = 0x704, |
dcd1fd72 MS |
404 | .reg_perf_state = 0x920, |
405 | .lut_row_size = 32, | |
406 | }; | |
407 | ||
49b59f4c MS |
408 | static const struct qcom_cpufreq_soc_data epss_soc_data = { |
409 | .reg_enable = 0x0, | |
f84ccad5 | 410 | .reg_domain_state = 0x20, |
c377d4ba | 411 | .reg_dcvs_ctrl = 0xb0, |
49b59f4c MS |
412 | .reg_freq_lut = 0x100, |
413 | .reg_volt_lut = 0x200, | |
e4e64486 | 414 | .reg_intr_clr = 0x308, |
49b59f4c MS |
415 | .reg_perf_state = 0x320, |
416 | .lut_row_size = 4, | |
417 | }; | |
418 | ||
dcd1fd72 MS |
419 | static const struct of_device_id qcom_cpufreq_hw_match[] = { |
420 | { .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data }, | |
49b59f4c | 421 | { .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data }, |
dcd1fd72 MS |
422 | {} |
423 | }; | |
424 | MODULE_DEVICE_TABLE(of, qcom_cpufreq_hw_match); | |
425 | ||
275157b3 TG |
426 | static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index) |
427 | { | |
428 | struct qcom_cpufreq_data *data = policy->driver_data; | |
429 | struct platform_device *pdev = cpufreq_get_driver_data(); | |
275157b3 TG |
430 | int ret; |
431 | ||
432 | /* | |
433 | * Look for LMh interrupt. If no interrupt line is specified / | |
434 | * if there is an error, allow cpufreq to be enabled as usual. | |
435 | */ | |
8f5783ad SB |
436 | data->throttle_irq = platform_get_irq_optional(pdev, index); |
437 | if (data->throttle_irq == -ENXIO) | |
438 | return 0; | |
439 | if (data->throttle_irq < 0) | |
440 | return data->throttle_irq; | |
275157b3 TG |
441 | |
442 | data->cancel_throttle = false; | |
443 | data->policy = policy; | |
444 | ||
445 | mutex_init(&data->throttle_lock); | |
446 | INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll); | |
447 | ||
be6592ed | 448 | snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu); |
275157b3 | 449 | ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq, |
ef8ee1cb | 450 | IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data); |
275157b3 | 451 | if (ret) { |
be6592ed | 452 | dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret); |
275157b3 TG |
453 | return 0; |
454 | } | |
455 | ||
f2b03dff | 456 | ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus); |
3ed6dfbd VZ |
457 | if (ret) |
458 | dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", | |
459 | data->irq_name, data->throttle_irq); | |
460 | ||
275157b3 TG |
461 | return 0; |
462 | } | |
463 | ||
a1eb080a | 464 | static int qcom_cpufreq_hw_cpu_online(struct cpufreq_policy *policy) |
275157b3 | 465 | { |
a1eb080a DB |
466 | struct qcom_cpufreq_data *data = policy->driver_data; |
467 | struct platform_device *pdev = cpufreq_get_driver_data(); | |
468 | int ret; | |
469 | ||
668a7a12 SB |
470 | if (data->throttle_irq <= 0) |
471 | return 0; | |
472 | ||
cdcf8eb3 PG |
473 | mutex_lock(&data->throttle_lock); |
474 | data->cancel_throttle = false; | |
475 | mutex_unlock(&data->throttle_lock); | |
476 | ||
f2b03dff | 477 | ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus); |
a1eb080a DB |
478 | if (ret) |
479 | dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n", | |
480 | data->irq_name, data->throttle_irq); | |
481 | ||
482 | return ret; | |
483 | } | |
484 | ||
485 | static int qcom_cpufreq_hw_cpu_offline(struct cpufreq_policy *policy) | |
486 | { | |
487 | struct qcom_cpufreq_data *data = policy->driver_data; | |
488 | ||
275157b3 | 489 | if (data->throttle_irq <= 0) |
a1eb080a | 490 | return 0; |
275157b3 TG |
491 | |
492 | mutex_lock(&data->throttle_lock); | |
493 | data->cancel_throttle = true; | |
494 | mutex_unlock(&data->throttle_lock); | |
495 | ||
496 | cancel_delayed_work_sync(&data->throttle_work); | |
f2b03dff | 497 | irq_set_affinity_and_hint(data->throttle_irq, NULL); |
f7fca54a | 498 | disable_irq_nosync(data->throttle_irq); |
a1eb080a DB |
499 | |
500 | return 0; | |
501 | } | |
502 | ||
503 | static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data) | |
504 | { | |
668a7a12 SB |
505 | if (data->throttle_irq <= 0) |
506 | return; | |
507 | ||
275157b3 TG |
508 | free_irq(data->throttle_irq, data); |
509 | } | |
510 | ||
2849dd8b TD |
511 | static int qcom_cpufreq_hw_cpu_init(struct cpufreq_policy *policy) |
512 | { | |
bd74e286 MS |
513 | struct platform_device *pdev = cpufreq_get_driver_data(); |
514 | struct device *dev = &pdev->dev; | |
2849dd8b TD |
515 | struct of_phandle_args args; |
516 | struct device_node *cpu_np; | |
55538fbc | 517 | struct device *cpu_dev; |
dcd1fd72 | 518 | struct qcom_cpufreq_data *data; |
2849dd8b TD |
519 | int ret, index; |
520 | ||
55538fbc TD |
521 | cpu_dev = get_cpu_device(policy->cpu); |
522 | if (!cpu_dev) { | |
523 | pr_err("%s: failed to get cpu%d device\n", __func__, | |
524 | policy->cpu); | |
525 | return -ENODEV; | |
526 | } | |
527 | ||
2849dd8b TD |
528 | cpu_np = of_cpu_device_node_get(policy->cpu); |
529 | if (!cpu_np) | |
530 | return -EINVAL; | |
531 | ||
532 | ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain", | |
533 | "#freq-domain-cells", 0, &args); | |
534 | of_node_put(cpu_np); | |
535 | if (ret) | |
536 | return ret; | |
537 | ||
538 | index = args.args[0]; | |
054a3ef6 | 539 | data = &qcom_cpufreq.data[index]; |
2849dd8b TD |
540 | |
541 | /* HW should be in enabled state to proceed */ | |
4f796170 | 542 | if (!(readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_enable) & 0x1)) { |
2849dd8b | 543 | dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index); |
054a3ef6 | 544 | return -ENODEV; |
2849dd8b TD |
545 | } |
546 | ||
4f796170 | 547 | if (readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_dcvs_ctrl) & 0x1) |
c377d4ba BA |
548 | data->per_core_dcvs = true; |
549 | ||
2849dd8b | 550 | qcom_get_related_cpus(index, policy->cpus); |
2849dd8b | 551 | |
dcd1fd72 | 552 | policy->driver_data = data; |
f0712ace | 553 | policy->dvfs_possible_from_any_cpu = true; |
2849dd8b | 554 | |
dcd1fd72 | 555 | ret = qcom_cpufreq_hw_read_lut(cpu_dev, policy); |
2849dd8b TD |
556 | if (ret) { |
557 | dev_err(dev, "Domain-%d failed to read LUT\n", index); | |
054a3ef6 | 558 | return ret; |
2849dd8b TD |
559 | } |
560 | ||
55538fbc TD |
561 | ret = dev_pm_opp_get_opp_count(cpu_dev); |
562 | if (ret <= 0) { | |
563 | dev_err(cpu_dev, "Failed to add OPPs\n"); | |
054a3ef6 | 564 | return -ENODEV; |
55538fbc TD |
565 | } |
566 | ||
26699172 SG |
567 | if (policy_has_boost_freq(policy)) { |
568 | ret = cpufreq_enable_boost_support(); | |
569 | if (ret) | |
570 | dev_warn(cpu_dev, "failed to enable boost: %d\n", ret); | |
571 | } | |
572 | ||
054a3ef6 | 573 | return qcom_cpufreq_hw_lmh_init(policy, index); |
2849dd8b TD |
574 | } |
575 | ||
576 | static int qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy) | |
577 | { | |
55538fbc | 578 | struct device *cpu_dev = get_cpu_device(policy->cpu); |
dcd1fd72 | 579 | struct qcom_cpufreq_data *data = policy->driver_data; |
2849dd8b | 580 | |
55538fbc | 581 | dev_pm_opp_remove_all_dynamic(cpu_dev); |
51c843cf | 582 | dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); |
275157b3 | 583 | qcom_cpufreq_hw_lmh_exit(data); |
2849dd8b | 584 | kfree(policy->freq_table); |
67fc209b | 585 | kfree(data); |
2849dd8b TD |
586 | |
587 | return 0; | |
588 | } | |
589 | ||
ef8ee1cb BA |
590 | static void qcom_cpufreq_ready(struct cpufreq_policy *policy) |
591 | { | |
592 | struct qcom_cpufreq_data *data = policy->driver_data; | |
593 | ||
594 | if (data->throttle_irq >= 0) | |
595 | enable_irq(data->throttle_irq); | |
596 | } | |
597 | ||
2849dd8b TD |
598 | static struct freq_attr *qcom_cpufreq_hw_attr[] = { |
599 | &cpufreq_freq_attr_scaling_available_freqs, | |
600 | &cpufreq_freq_attr_scaling_boost_freqs, | |
601 | NULL | |
602 | }; | |
603 | ||
604 | static struct cpufreq_driver cpufreq_qcom_hw_driver = { | |
5ae4a4b4 | 605 | .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK | |
4c5ff1c8 AK |
606 | CPUFREQ_HAVE_GOVERNOR_PER_POLICY | |
607 | CPUFREQ_IS_COOLING_DEV, | |
2849dd8b TD |
608 | .verify = cpufreq_generic_frequency_table_verify, |
609 | .target_index = qcom_cpufreq_hw_target_index, | |
610 | .get = qcom_cpufreq_hw_get, | |
611 | .init = qcom_cpufreq_hw_cpu_init, | |
612 | .exit = qcom_cpufreq_hw_cpu_exit, | |
a1eb080a DB |
613 | .online = qcom_cpufreq_hw_cpu_online, |
614 | .offline = qcom_cpufreq_hw_cpu_offline, | |
e96c2153 | 615 | .register_em = cpufreq_register_em_with_opp, |
2849dd8b TD |
616 | .fast_switch = qcom_cpufreq_hw_fast_switch, |
617 | .name = "qcom-cpufreq-hw", | |
618 | .attr = qcom_cpufreq_hw_attr, | |
ef8ee1cb | 619 | .ready = qcom_cpufreq_ready, |
2849dd8b TD |
620 | }; |
621 | ||
4370232c MS |
622 | static unsigned long qcom_cpufreq_hw_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) |
623 | { | |
624 | struct qcom_cpufreq_data *data = container_of(hw, struct qcom_cpufreq_data, cpu_clk); | |
625 | ||
626 | return qcom_lmh_get_throttle_freq(data); | |
627 | } | |
628 | ||
629 | static const struct clk_ops qcom_cpufreq_hw_clk_ops = { | |
630 | .recalc_rate = qcom_cpufreq_hw_recalc_rate, | |
631 | }; | |
632 | ||
2849dd8b TD |
633 | static int qcom_cpufreq_hw_driver_probe(struct platform_device *pdev) |
634 | { | |
4370232c | 635 | struct clk_hw_onecell_data *clk_data; |
7cfa8553 | 636 | struct device *dev = &pdev->dev; |
51c843cf | 637 | struct device *cpu_dev; |
2849dd8b | 638 | struct clk *clk; |
35527c67 | 639 | int ret, i, num_domains; |
2849dd8b | 640 | |
7cfa8553 | 641 | clk = clk_get(dev, "xo"); |
2849dd8b TD |
642 | if (IS_ERR(clk)) |
643 | return PTR_ERR(clk); | |
644 | ||
645 | xo_rate = clk_get_rate(clk); | |
646 | clk_put(clk); | |
647 | ||
7cfa8553 | 648 | clk = clk_get(dev, "alternate"); |
2849dd8b TD |
649 | if (IS_ERR(clk)) |
650 | return PTR_ERR(clk); | |
651 | ||
652 | cpu_hw_rate = clk_get_rate(clk) / CLK_HW_DIV; | |
653 | clk_put(clk); | |
654 | ||
bd74e286 | 655 | cpufreq_qcom_hw_driver.driver_data = pdev; |
2849dd8b | 656 | |
51c843cf SS |
657 | /* Check for optional interconnect paths on CPU0 */ |
658 | cpu_dev = get_cpu_device(0); | |
659 | if (!cpu_dev) | |
660 | return -EPROBE_DEFER; | |
661 | ||
662 | ret = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); | |
663 | if (ret) | |
5ee64250 | 664 | return dev_err_probe(dev, ret, "Failed to find icc paths\n"); |
51c843cf | 665 | |
35527c67 KD |
666 | for (num_domains = 0; num_domains < MAX_FREQ_DOMAINS; num_domains++) |
667 | if (!platform_get_resource(pdev, IORESOURCE_MEM, num_domains)) | |
668 | break; | |
054a3ef6 | 669 | |
7cfa8553 | 670 | qcom_cpufreq.data = devm_kzalloc(dev, sizeof(struct qcom_cpufreq_data) * num_domains, |
054a3ef6 MS |
671 | GFP_KERNEL); |
672 | if (!qcom_cpufreq.data) | |
673 | return -ENOMEM; | |
674 | ||
4f796170 | 675 | qcom_cpufreq.soc_data = of_device_get_match_data(dev); |
6f098cde MS |
676 | if (!qcom_cpufreq.soc_data) |
677 | return -ENODEV; | |
4f796170 | 678 | |
4370232c MS |
679 | clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, num_domains), GFP_KERNEL); |
680 | if (!clk_data) | |
681 | return -ENOMEM; | |
682 | ||
683 | clk_data->num = num_domains; | |
684 | ||
054a3ef6 MS |
685 | for (i = 0; i < num_domains; i++) { |
686 | struct qcom_cpufreq_data *data = &qcom_cpufreq.data[i]; | |
4370232c | 687 | struct clk_init_data clk_init = {}; |
054a3ef6 MS |
688 | void __iomem *base; |
689 | ||
ba5e770c | 690 | base = devm_platform_ioremap_resource(pdev, i); |
054a3ef6 | 691 | if (IS_ERR(base)) { |
ba5e770c | 692 | dev_err(dev, "Failed to map resource index %d\n", i); |
054a3ef6 MS |
693 | return PTR_ERR(base); |
694 | } | |
695 | ||
696 | data->base = base; | |
4370232c MS |
697 | |
698 | /* Register CPU clock for each frequency domain */ | |
699 | clk_init.name = kasprintf(GFP_KERNEL, "qcom_cpufreq%d", i); | |
700 | if (!clk_init.name) | |
701 | return -ENOMEM; | |
702 | ||
703 | clk_init.flags = CLK_GET_RATE_NOCACHE; | |
704 | clk_init.ops = &qcom_cpufreq_hw_clk_ops; | |
705 | data->cpu_clk.init = &clk_init; | |
706 | ||
707 | ret = devm_clk_hw_register(dev, &data->cpu_clk); | |
708 | if (ret < 0) { | |
709 | dev_err(dev, "Failed to register clock %d: %d\n", i, ret); | |
710 | kfree(clk_init.name); | |
711 | return ret; | |
712 | } | |
713 | ||
714 | clk_data->hws[i] = &data->cpu_clk; | |
715 | kfree(clk_init.name); | |
716 | } | |
717 | ||
718 | ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data); | |
719 | if (ret < 0) { | |
720 | dev_err(dev, "Failed to add clock provider\n"); | |
721 | return ret; | |
054a3ef6 MS |
722 | } |
723 | ||
2849dd8b TD |
724 | ret = cpufreq_register_driver(&cpufreq_qcom_hw_driver); |
725 | if (ret) | |
7cfa8553 | 726 | dev_err(dev, "CPUFreq HW driver failed to register\n"); |
2849dd8b | 727 | else |
7cfa8553 | 728 | dev_dbg(dev, "QCOM CPUFreq HW driver initialized\n"); |
2849dd8b TD |
729 | |
730 | return ret; | |
731 | } | |
732 | ||
573d54db | 733 | static void qcom_cpufreq_hw_driver_remove(struct platform_device *pdev) |
2849dd8b | 734 | { |
dd329e1e | 735 | cpufreq_unregister_driver(&cpufreq_qcom_hw_driver); |
2849dd8b TD |
736 | } |
737 | ||
2849dd8b TD |
738 | static struct platform_driver qcom_cpufreq_hw_driver = { |
739 | .probe = qcom_cpufreq_hw_driver_probe, | |
573d54db | 740 | .remove_new = qcom_cpufreq_hw_driver_remove, |
2849dd8b TD |
741 | .driver = { |
742 | .name = "qcom-cpufreq-hw", | |
743 | .of_match_table = qcom_cpufreq_hw_match, | |
744 | }, | |
745 | }; | |
746 | ||
747 | static int __init qcom_cpufreq_hw_init(void) | |
748 | { | |
749 | return platform_driver_register(&qcom_cpufreq_hw_driver); | |
750 | } | |
11ff4bdd | 751 | postcore_initcall(qcom_cpufreq_hw_init); |
2849dd8b TD |
752 | |
753 | static void __exit qcom_cpufreq_hw_exit(void) | |
754 | { | |
755 | platform_driver_unregister(&qcom_cpufreq_hw_driver); | |
756 | } | |
757 | module_exit(qcom_cpufreq_hw_exit); | |
758 | ||
759 | MODULE_DESCRIPTION("QCOM CPUFREQ HW Driver"); | |
760 | MODULE_LICENSE("GPL v2"); |