Commit | Line | Data |
---|---|---|
b886d83c | 1 | // SPDX-License-Identifier: GPL-2.0-only |
5477fb3b AC |
2 | /* |
3 | * CPPC (Collaborative Processor Performance Control) driver for | |
4 | * interfacing with the CPUfreq layer and governors. See | |
5 | * cppc_acpi.c for CPPC specific methods. | |
6 | * | |
7 | * (C) Copyright 2014, 2015 Linaro Ltd. | |
8 | * Author: Ashwin Chaugule <ashwin.chaugule@linaro.org> | |
5477fb3b AC |
9 | */ |
10 | ||
11 | #define pr_fmt(fmt) "CPPC Cpufreq:" fmt | |
12 | ||
1eb5dde6 | 13 | #include <linux/arch_topology.h> |
5477fb3b AC |
14 | #include <linux/kernel.h> |
15 | #include <linux/module.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/cpu.h> | |
18 | #include <linux/cpufreq.h> | |
1eb5dde6 VK |
19 | #include <linux/irq_work.h> |
20 | #include <linux/kthread.h> | |
3d41386d | 21 | #include <linux/time.h> |
5477fb3b | 22 | #include <linux/vmalloc.h> |
1eb5dde6 | 23 | #include <uapi/linux/sched/types.h> |
5477fb3b | 24 | |
ad38677d AS |
25 | #include <asm/unaligned.h> |
26 | ||
5477fb3b AC |
27 | #include <acpi/cppc_acpi.h> |
28 | ||
29 | /* | |
a28b2bfc IV |
30 | * This list contains information parsed from per CPU ACPI _CPC and _PSD |
31 | * structures: e.g. the highest and lowest supported performance, capabilities, | |
32 | * desired performance, level requested etc. Depending on the share_type, not | |
33 | * all CPUs will have an entry in the list. | |
5477fb3b | 34 | */ |
a28b2bfc IV |
35 | static LIST_HEAD(cpu_data_list); |
36 | ||
54e74df5 | 37 | static bool boost_supported; |
5477fb3b | 38 | |
6c8d750f | 39 | struct cppc_workaround_oem_info { |
c7402379 | 40 | char oem_id[ACPI_OEM_ID_SIZE + 1]; |
6c8d750f XW |
41 | char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; |
42 | u32 oem_revision; | |
43 | }; | |
44 | ||
6c8d750f XW |
45 | static struct cppc_workaround_oem_info wa_info[] = { |
46 | { | |
47 | .oem_id = "HISI ", | |
48 | .oem_table_id = "HIP07 ", | |
49 | .oem_revision = 0, | |
50 | }, { | |
51 | .oem_id = "HISI ", | |
52 | .oem_table_id = "HIP08 ", | |
53 | .oem_revision = 0, | |
54 | } | |
55 | }; | |
56 | ||
a3f083e0 ZB |
57 | static struct cpufreq_driver cppc_cpufreq_driver; |
58 | ||
ae2df912 JL |
59 | static enum { |
60 | FIE_UNSET = -1, | |
61 | FIE_ENABLED, | |
62 | FIE_DISABLED | |
63 | } fie_disabled = FIE_UNSET; | |
64 | ||
1eb5dde6 | 65 | #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE |
ae2df912 JL |
66 | module_param(fie_disabled, int, 0444); |
67 | MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)"); | |
1eb5dde6 VK |
68 | |
69 | /* Frequency invariance support */ | |
70 | struct cppc_freq_invariance { | |
71 | int cpu; | |
72 | struct irq_work irq_work; | |
73 | struct kthread_work work; | |
74 | struct cppc_perf_fb_ctrs prev_perf_fb_ctrs; | |
75 | struct cppc_cpudata *cpu_data; | |
76 | }; | |
77 | ||
78 | static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_freq_inv); | |
79 | static struct kthread_worker *kworker_fie; | |
80 | ||
1eb5dde6 VK |
81 | static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu); |
82 | static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, | |
83 | struct cppc_perf_fb_ctrs *fb_ctrs_t0, | |
84 | struct cppc_perf_fb_ctrs *fb_ctrs_t1); | |
85 | ||
86 | /** | |
87 | * cppc_scale_freq_workfn - CPPC arch_freq_scale updater for frequency invariance | |
88 | * @work: The work item. | |
89 | * | |
90 | * The CPPC driver register itself with the topology core to provide its own | |
91 | * implementation (cppc_scale_freq_tick()) of topology_scale_freq_tick() which | |
92 | * gets called by the scheduler on every tick. | |
93 | * | |
94 | * Note that the arch specific counters have higher priority than CPPC counters, | |
95 | * if available, though the CPPC driver doesn't need to have any special | |
96 | * handling for that. | |
97 | * | |
98 | * On an invocation of cppc_scale_freq_tick(), we schedule an irq work (since we | |
99 | * reach here from hard-irq context), which then schedules a normal work item | |
100 | * and cppc_scale_freq_workfn() updates the per_cpu arch_freq_scale variable | |
101 | * based on the counter updates since the last tick. | |
102 | */ | |
103 | static void cppc_scale_freq_workfn(struct kthread_work *work) | |
104 | { | |
105 | struct cppc_freq_invariance *cppc_fi; | |
106 | struct cppc_perf_fb_ctrs fb_ctrs = {0}; | |
107 | struct cppc_cpudata *cpu_data; | |
108 | unsigned long local_freq_scale; | |
109 | u64 perf; | |
110 | ||
111 | cppc_fi = container_of(work, struct cppc_freq_invariance, work); | |
112 | cpu_data = cppc_fi->cpu_data; | |
113 | ||
114 | if (cppc_get_perf_ctrs(cppc_fi->cpu, &fb_ctrs)) { | |
115 | pr_warn("%s: failed to read perf counters\n", __func__); | |
116 | return; | |
117 | } | |
118 | ||
119 | perf = cppc_perf_from_fbctrs(cpu_data, &cppc_fi->prev_perf_fb_ctrs, | |
120 | &fb_ctrs); | |
121 | cppc_fi->prev_perf_fb_ctrs = fb_ctrs; | |
122 | ||
123 | perf <<= SCHED_CAPACITY_SHIFT; | |
124 | local_freq_scale = div64_u64(perf, cpu_data->perf_caps.highest_perf); | |
125 | ||
126 | /* This can happen due to counter's overflow */ | |
127 | if (unlikely(local_freq_scale > 1024)) | |
128 | local_freq_scale = 1024; | |
129 | ||
130 | per_cpu(arch_freq_scale, cppc_fi->cpu) = local_freq_scale; | |
131 | } | |
132 | ||
133 | static void cppc_irq_work(struct irq_work *irq_work) | |
134 | { | |
135 | struct cppc_freq_invariance *cppc_fi; | |
136 | ||
137 | cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work); | |
138 | kthread_queue_work(kworker_fie, &cppc_fi->work); | |
139 | } | |
140 | ||
141 | static void cppc_scale_freq_tick(void) | |
142 | { | |
143 | struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_freq_inv, smp_processor_id()); | |
144 | ||
145 | /* | |
146 | * cppc_get_perf_ctrs() can potentially sleep, call that from the right | |
147 | * context. | |
148 | */ | |
149 | irq_work_queue(&cppc_fi->irq_work); | |
150 | } | |
151 | ||
152 | static struct scale_freq_data cppc_sftd = { | |
153 | .source = SCALE_FREQ_SOURCE_CPPC, | |
154 | .set_freq_scale = cppc_scale_freq_tick, | |
155 | }; | |
156 | ||
157 | static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) | |
158 | { | |
159 | struct cppc_freq_invariance *cppc_fi; | |
160 | int cpu, ret; | |
161 | ||
ae2df912 | 162 | if (fie_disabled) |
1eb5dde6 VK |
163 | return; |
164 | ||
165 | for_each_cpu(cpu, policy->cpus) { | |
166 | cppc_fi = &per_cpu(cppc_freq_inv, cpu); | |
167 | cppc_fi->cpu = cpu; | |
168 | cppc_fi->cpu_data = policy->driver_data; | |
169 | kthread_init_work(&cppc_fi->work, cppc_scale_freq_workfn); | |
170 | init_irq_work(&cppc_fi->irq_work, cppc_irq_work); | |
171 | ||
172 | ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs); | |
173 | if (ret) { | |
174 | pr_warn("%s: failed to read perf counters for cpu:%d: %d\n", | |
175 | __func__, cpu, ret); | |
176 | ||
177 | /* | |
178 | * Don't abort if the CPU was offline while the driver | |
179 | * was getting registered. | |
180 | */ | |
181 | if (cpu_online(cpu)) | |
182 | return; | |
183 | } | |
184 | } | |
185 | ||
186 | /* Register for freq-invariance */ | |
187 | topology_set_scale_freq_source(&cppc_sftd, policy->cpus); | |
188 | } | |
189 | ||
190 | /* | |
191 | * We free all the resources on policy's removal and not on CPU removal as the | |
192 | * irq-work are per-cpu and the hotplug core takes care of flushing the pending | |
193 | * irq-works (hint: smpcfd_dying_cpu()) on CPU hotplug. Even if the kthread-work | |
194 | * fires on another CPU after the concerned CPU is removed, it won't harm. | |
195 | * | |
196 | * We just need to make sure to remove them all on policy->exit(). | |
197 | */ | |
198 | static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) | |
199 | { | |
200 | struct cppc_freq_invariance *cppc_fi; | |
201 | int cpu; | |
202 | ||
ae2df912 | 203 | if (fie_disabled) |
1eb5dde6 VK |
204 | return; |
205 | ||
206 | /* policy->cpus will be empty here, use related_cpus instead */ | |
207 | topology_clear_scale_freq_source(SCALE_FREQ_SOURCE_CPPC, policy->related_cpus); | |
208 | ||
209 | for_each_cpu(cpu, policy->related_cpus) { | |
210 | cppc_fi = &per_cpu(cppc_freq_inv, cpu); | |
211 | irq_work_sync(&cppc_fi->irq_work); | |
212 | kthread_cancel_work_sync(&cppc_fi->work); | |
213 | } | |
214 | } | |
215 | ||
216 | static void __init cppc_freq_invariance_init(void) | |
217 | { | |
218 | struct sched_attr attr = { | |
219 | .size = sizeof(struct sched_attr), | |
220 | .sched_policy = SCHED_DEADLINE, | |
221 | .sched_nice = 0, | |
222 | .sched_priority = 0, | |
223 | /* | |
224 | * Fake (unused) bandwidth; workaround to "fix" | |
225 | * priority inheritance. | |
226 | */ | |
227 | .sched_runtime = 1000000, | |
228 | .sched_deadline = 10000000, | |
229 | .sched_period = 10000000, | |
230 | }; | |
231 | int ret; | |
232 | ||
ae2df912 JL |
233 | if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) { |
234 | fie_disabled = FIE_ENABLED; | |
235 | if (cppc_perf_ctrs_in_pcc()) { | |
236 | pr_info("FIE not enabled on systems with registers in PCC\n"); | |
237 | fie_disabled = FIE_DISABLED; | |
238 | } | |
239 | } | |
240 | ||
241 | if (fie_disabled) | |
1eb5dde6 VK |
242 | return; |
243 | ||
244 | kworker_fie = kthread_create_worker(0, "cppc_fie"); | |
e613d8cf LC |
245 | if (IS_ERR(kworker_fie)) { |
246 | pr_warn("%s: failed to create kworker_fie: %ld\n", __func__, | |
247 | PTR_ERR(kworker_fie)); | |
248 | fie_disabled = FIE_DISABLED; | |
1eb5dde6 | 249 | return; |
e613d8cf | 250 | } |
1eb5dde6 VK |
251 | |
252 | ret = sched_setattr_nocheck(kworker_fie->task, &attr); | |
253 | if (ret) { | |
254 | pr_warn("%s: failed to set SCHED_DEADLINE: %d\n", __func__, | |
255 | ret); | |
256 | kthread_destroy_worker(kworker_fie); | |
e613d8cf | 257 | fie_disabled = FIE_DISABLED; |
1eb5dde6 VK |
258 | } |
259 | } | |
260 | ||
261 | static void cppc_freq_invariance_exit(void) | |
262 | { | |
ae2df912 | 263 | if (fie_disabled) |
1eb5dde6 VK |
264 | return; |
265 | ||
266 | kthread_destroy_worker(kworker_fie); | |
1eb5dde6 VK |
267 | } |
268 | ||
269 | #else | |
270 | static inline void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) | |
271 | { | |
272 | } | |
273 | ||
274 | static inline void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy) | |
275 | { | |
276 | } | |
277 | ||
278 | static inline void cppc_freq_invariance_init(void) | |
279 | { | |
280 | } | |
281 | ||
282 | static inline void cppc_freq_invariance_exit(void) | |
283 | { | |
284 | } | |
285 | #endif /* CONFIG_ACPI_CPPC_CPUFREQ_FIE */ | |
286 | ||
5477fb3b | 287 | static int cppc_cpufreq_set_target(struct cpufreq_policy *policy, |
63087265 IV |
288 | unsigned int target_freq, |
289 | unsigned int relation) | |
5477fb3b | 290 | { |
a28b2bfc | 291 | struct cppc_cpudata *cpu_data = policy->driver_data; |
d2641a5c | 292 | unsigned int cpu = policy->cpu; |
5477fb3b | 293 | struct cpufreq_freqs freqs; |
c197d758 | 294 | u32 desired_perf; |
5477fb3b AC |
295 | int ret = 0; |
296 | ||
50b813b1 | 297 | desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); |
c197d758 | 298 | /* Return if it is exactly the same perf */ |
48ad8dc9 | 299 | if (desired_perf == cpu_data->perf_ctrls.desired_perf) |
c197d758 HT |
300 | return ret; |
301 | ||
48ad8dc9 | 302 | cpu_data->perf_ctrls.desired_perf = desired_perf; |
5477fb3b AC |
303 | freqs.old = policy->cur; |
304 | freqs.new = target_freq; | |
305 | ||
306 | cpufreq_freq_transition_begin(policy, &freqs); | |
d2641a5c | 307 | ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); |
5477fb3b AC |
308 | cpufreq_freq_transition_end(policy, &freqs, ret != 0); |
309 | ||
310 | if (ret) | |
311 | pr_debug("Failed to set target on CPU:%d. ret:%d\n", | |
d2641a5c | 312 | cpu, ret); |
5477fb3b AC |
313 | |
314 | return ret; | |
315 | } | |
316 | ||
3cc30dd0 PG |
317 | static unsigned int cppc_cpufreq_fast_switch(struct cpufreq_policy *policy, |
318 | unsigned int target_freq) | |
319 | { | |
320 | struct cppc_cpudata *cpu_data = policy->driver_data; | |
321 | unsigned int cpu = policy->cpu; | |
322 | u32 desired_perf; | |
323 | int ret; | |
324 | ||
50b813b1 | 325 | desired_perf = cppc_khz_to_perf(&cpu_data->perf_caps, target_freq); |
3cc30dd0 PG |
326 | cpu_data->perf_ctrls.desired_perf = desired_perf; |
327 | ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); | |
328 | ||
329 | if (ret) { | |
330 | pr_debug("Failed to set target on CPU:%d. ret:%d\n", | |
331 | cpu, ret); | |
332 | return 0; | |
333 | } | |
334 | ||
335 | return target_freq; | |
336 | } | |
337 | ||
1e4f63ae | 338 | static int cppc_verify_policy(struct cpufreq_policy_data *policy) |
5477fb3b AC |
339 | { |
340 | cpufreq_verify_within_cpu_limits(policy); | |
341 | return 0; | |
342 | } | |
343 | ||
d4f3388a PP |
344 | /* |
345 | * The PCC subspace describes the rate at which platform can accept commands | |
346 | * on the shared PCC channel (including READs which do not count towards freq | |
63087265 | 347 | * transition requests), so ideally we need to use the PCC values as a fallback |
d4f3388a PP |
348 | * if we don't have a platform specific transition_delay_us |
349 | */ | |
350 | #ifdef CONFIG_ARM64 | |
351 | #include <asm/cputype.h> | |
352 | ||
48ad8dc9 | 353 | static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) |
d4f3388a PP |
354 | { |
355 | unsigned long implementor = read_cpuid_implementor(); | |
356 | unsigned long part_num = read_cpuid_part_number(); | |
d4f3388a PP |
357 | |
358 | switch (implementor) { | |
359 | case ARM_CPU_IMP_QCOM: | |
360 | switch (part_num) { | |
361 | case QCOM_CPU_PART_FALKOR_V1: | |
362 | case QCOM_CPU_PART_FALKOR: | |
2b53d1bd | 363 | return 10000; |
d4f3388a | 364 | } |
d4f3388a | 365 | } |
2b53d1bd | 366 | return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; |
d4f3388a | 367 | } |
da436345 PG |
368 | #else |
369 | static unsigned int cppc_cpufreq_get_transition_delay_us(unsigned int cpu) | |
370 | { | |
371 | return cppc_get_transition_latency(cpu) / NSEC_PER_USEC; | |
372 | } | |
373 | #endif | |
374 | ||
375 | #if defined(CONFIG_ARM64) && defined(CONFIG_ENERGY_MODEL) | |
d4f3388a | 376 | |
d3c3db41 | 377 | static DEFINE_PER_CPU(unsigned int, efficiency_class); |
740fcdc2 PG |
378 | static void cppc_cpufreq_register_em(struct cpufreq_policy *policy); |
379 | ||
380 | /* Create an artificial performance state every CPPC_EM_CAP_STEP capacity unit. */ | |
381 | #define CPPC_EM_CAP_STEP (20) | |
382 | /* Increase the cost value by CPPC_EM_COST_STEP every performance state. */ | |
383 | #define CPPC_EM_COST_STEP (1) | |
384 | /* Add a cost gap correspnding to the energy of 4 CPUs. */ | |
385 | #define CPPC_EM_COST_GAP (4 * SCHED_CAPACITY_SCALE * CPPC_EM_COST_STEP \ | |
386 | / CPPC_EM_CAP_STEP) | |
387 | ||
388 | static unsigned int get_perf_level_count(struct cpufreq_policy *policy) | |
389 | { | |
390 | struct cppc_perf_caps *perf_caps; | |
391 | unsigned int min_cap, max_cap; | |
392 | struct cppc_cpudata *cpu_data; | |
393 | int cpu = policy->cpu; | |
394 | ||
395 | cpu_data = policy->driver_data; | |
396 | perf_caps = &cpu_data->perf_caps; | |
397 | max_cap = arch_scale_cpu_capacity(cpu); | |
f5f94b9c PG |
398 | min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, |
399 | perf_caps->highest_perf); | |
740fcdc2 PG |
400 | if ((min_cap == 0) || (max_cap < min_cap)) |
401 | return 0; | |
402 | return 1 + max_cap / CPPC_EM_CAP_STEP - min_cap / CPPC_EM_CAP_STEP; | |
403 | } | |
404 | ||
405 | /* | |
406 | * The cost is defined as: | |
407 | * cost = power * max_frequency / frequency | |
408 | */ | |
409 | static inline unsigned long compute_cost(int cpu, int step) | |
410 | { | |
411 | return CPPC_EM_COST_GAP * per_cpu(efficiency_class, cpu) + | |
412 | step * CPPC_EM_COST_STEP; | |
413 | } | |
414 | ||
415 | static int cppc_get_cpu_power(struct device *cpu_dev, | |
416 | unsigned long *power, unsigned long *KHz) | |
417 | { | |
418 | unsigned long perf_step, perf_prev, perf, perf_check; | |
419 | unsigned int min_step, max_step, step, step_check; | |
420 | unsigned long prev_freq = *KHz; | |
421 | unsigned int min_cap, max_cap; | |
422 | struct cpufreq_policy *policy; | |
423 | ||
424 | struct cppc_perf_caps *perf_caps; | |
425 | struct cppc_cpudata *cpu_data; | |
426 | ||
427 | policy = cpufreq_cpu_get_raw(cpu_dev->id); | |
428 | cpu_data = policy->driver_data; | |
429 | perf_caps = &cpu_data->perf_caps; | |
430 | max_cap = arch_scale_cpu_capacity(cpu_dev->id); | |
f5f94b9c PG |
431 | min_cap = div_u64((u64)max_cap * perf_caps->lowest_perf, |
432 | perf_caps->highest_perf); | |
433 | perf_step = div_u64((u64)CPPC_EM_CAP_STEP * perf_caps->highest_perf, | |
434 | max_cap); | |
740fcdc2 PG |
435 | min_step = min_cap / CPPC_EM_CAP_STEP; |
436 | max_step = max_cap / CPPC_EM_CAP_STEP; | |
437 | ||
50b813b1 | 438 | perf_prev = cppc_khz_to_perf(perf_caps, *KHz); |
740fcdc2 PG |
439 | step = perf_prev / perf_step; |
440 | ||
441 | if (step > max_step) | |
442 | return -EINVAL; | |
443 | ||
444 | if (min_step == max_step) { | |
445 | step = max_step; | |
446 | perf = perf_caps->highest_perf; | |
447 | } else if (step < min_step) { | |
448 | step = min_step; | |
449 | perf = perf_caps->lowest_perf; | |
450 | } else { | |
451 | step++; | |
452 | if (step == max_step) | |
453 | perf = perf_caps->highest_perf; | |
454 | else | |
455 | perf = step * perf_step; | |
456 | } | |
457 | ||
50b813b1 VG |
458 | *KHz = cppc_perf_to_khz(perf_caps, perf); |
459 | perf_check = cppc_khz_to_perf(perf_caps, *KHz); | |
740fcdc2 PG |
460 | step_check = perf_check / perf_step; |
461 | ||
462 | /* | |
463 | * To avoid bad integer approximation, check that new frequency value | |
464 | * increased and that the new frequency will be converted to the | |
465 | * desired step value. | |
466 | */ | |
467 | while ((*KHz == prev_freq) || (step_check != step)) { | |
468 | perf++; | |
50b813b1 VG |
469 | *KHz = cppc_perf_to_khz(perf_caps, perf); |
470 | perf_check = cppc_khz_to_perf(perf_caps, *KHz); | |
740fcdc2 PG |
471 | step_check = perf_check / perf_step; |
472 | } | |
473 | ||
474 | /* | |
475 | * With an artificial EM, only the cost value is used. Still the power | |
476 | * is populated such as 0 < power < EM_MAX_POWER. This allows to add | |
477 | * more sense to the artificial performance states. | |
478 | */ | |
479 | *power = compute_cost(cpu_dev->id, step); | |
480 | ||
481 | return 0; | |
482 | } | |
483 | ||
484 | static int cppc_get_cpu_cost(struct device *cpu_dev, unsigned long KHz, | |
485 | unsigned long *cost) | |
486 | { | |
487 | unsigned long perf_step, perf_prev; | |
488 | struct cppc_perf_caps *perf_caps; | |
489 | struct cpufreq_policy *policy; | |
490 | struct cppc_cpudata *cpu_data; | |
491 | unsigned int max_cap; | |
492 | int step; | |
493 | ||
494 | policy = cpufreq_cpu_get_raw(cpu_dev->id); | |
495 | cpu_data = policy->driver_data; | |
496 | perf_caps = &cpu_data->perf_caps; | |
497 | max_cap = arch_scale_cpu_capacity(cpu_dev->id); | |
498 | ||
50b813b1 | 499 | perf_prev = cppc_khz_to_perf(perf_caps, KHz); |
740fcdc2 PG |
500 | perf_step = CPPC_EM_CAP_STEP * perf_caps->highest_perf / max_cap; |
501 | step = perf_prev / perf_step; | |
502 | ||
503 | *cost = compute_cost(cpu_dev->id, step); | |
504 | ||
505 | return 0; | |
506 | } | |
d3c3db41 PG |
507 | |
508 | static int populate_efficiency_class(void) | |
509 | { | |
510 | struct acpi_madt_generic_interrupt *gicc; | |
511 | DECLARE_BITMAP(used_classes, 256) = {}; | |
512 | int class, cpu, index; | |
513 | ||
514 | for_each_possible_cpu(cpu) { | |
515 | gicc = acpi_cpu_get_madt_gicc(cpu); | |
516 | class = gicc->efficiency_class; | |
517 | bitmap_set(used_classes, class, 1); | |
518 | } | |
519 | ||
520 | if (bitmap_weight(used_classes, 256) <= 1) { | |
521 | pr_debug("Efficiency classes are all equal (=%d). " | |
522 | "No EM registered", class); | |
523 | return -EINVAL; | |
524 | } | |
525 | ||
526 | /* | |
527 | * Squeeze efficiency class values on [0:#efficiency_class-1]. | |
528 | * Values are per spec in [0:255]. | |
529 | */ | |
530 | index = 0; | |
531 | for_each_set_bit(class, used_classes, 256) { | |
532 | for_each_possible_cpu(cpu) { | |
533 | gicc = acpi_cpu_get_madt_gicc(cpu); | |
534 | if (gicc->efficiency_class == class) | |
535 | per_cpu(efficiency_class, cpu) = index; | |
536 | } | |
537 | index++; | |
538 | } | |
740fcdc2 | 539 | cppc_cpufreq_driver.register_em = cppc_cpufreq_register_em; |
d3c3db41 PG |
540 | |
541 | return 0; | |
542 | } | |
543 | ||
740fcdc2 PG |
544 | static void cppc_cpufreq_register_em(struct cpufreq_policy *policy) |
545 | { | |
546 | struct cppc_cpudata *cpu_data; | |
547 | struct em_data_callback em_cb = | |
548 | EM_ADV_DATA_CB(cppc_get_cpu_power, cppc_get_cpu_cost); | |
549 | ||
550 | cpu_data = policy->driver_data; | |
551 | em_dev_register_perf_domain(get_cpu_device(policy->cpu), | |
552 | get_perf_level_count(policy), &em_cb, | |
553 | cpu_data->shared_cpu_map, 0); | |
554 | } | |
555 | ||
d4f3388a | 556 | #else |
d3c3db41 PG |
557 | static int populate_efficiency_class(void) |
558 | { | |
559 | return 0; | |
560 | } | |
d4f3388a PP |
561 | #endif |
562 | ||
a28b2bfc | 563 | static struct cppc_cpudata *cppc_cpufreq_get_cpu_data(unsigned int cpu) |
5477fb3b | 564 | { |
a28b2bfc IV |
565 | struct cppc_cpudata *cpu_data; |
566 | int ret; | |
567 | ||
568 | cpu_data = kzalloc(sizeof(struct cppc_cpudata), GFP_KERNEL); | |
569 | if (!cpu_data) | |
570 | goto out; | |
5477fb3b | 571 | |
a28b2bfc IV |
572 | if (!zalloc_cpumask_var(&cpu_data->shared_cpu_map, GFP_KERNEL)) |
573 | goto free_cpu; | |
5477fb3b | 574 | |
a28b2bfc | 575 | ret = acpi_get_psd_map(cpu, cpu_data); |
5477fb3b | 576 | if (ret) { |
a28b2bfc IV |
577 | pr_debug("Err parsing CPU%d PSD data: ret:%d\n", cpu, ret); |
578 | goto free_mask; | |
579 | } | |
580 | ||
581 | ret = cppc_get_perf_caps(cpu, &cpu_data->perf_caps); | |
582 | if (ret) { | |
583 | pr_debug("Err reading CPU%d perf caps: ret:%d\n", cpu, ret); | |
584 | goto free_mask; | |
5477fb3b AC |
585 | } |
586 | ||
a28b2bfc IV |
587 | list_add(&cpu_data->node, &cpu_data_list); |
588 | ||
589 | return cpu_data; | |
590 | ||
591 | free_mask: | |
592 | free_cpumask_var(cpu_data->shared_cpu_map); | |
593 | free_cpu: | |
594 | kfree(cpu_data); | |
595 | out: | |
596 | return NULL; | |
597 | } | |
598 | ||
fe2535a4 VK |
599 | static void cppc_cpufreq_put_cpu_data(struct cpufreq_policy *policy) |
600 | { | |
601 | struct cppc_cpudata *cpu_data = policy->driver_data; | |
602 | ||
603 | list_del(&cpu_data->node); | |
604 | free_cpumask_var(cpu_data->shared_cpu_map); | |
605 | kfree(cpu_data); | |
606 | policy->driver_data = NULL; | |
607 | } | |
608 | ||
a28b2bfc IV |
609 | static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy) |
610 | { | |
611 | unsigned int cpu = policy->cpu; | |
612 | struct cppc_cpudata *cpu_data; | |
613 | struct cppc_perf_caps *caps; | |
614 | int ret; | |
615 | ||
616 | cpu_data = cppc_cpufreq_get_cpu_data(cpu); | |
617 | if (!cpu_data) { | |
618 | pr_err("Error in acquiring _CPC/_PSD data for CPU%d.\n", cpu); | |
619 | return -ENODEV; | |
620 | } | |
621 | caps = &cpu_data->perf_caps; | |
622 | policy->driver_data = cpu_data; | |
ad38677d | 623 | |
73808d0f PP |
624 | /* |
625 | * Set min to lowest nonlinear perf to avoid any efficiency penalty (see | |
626 | * Section 8.4.7.1.1.5 of ACPI 6.1 spec) | |
627 | */ | |
50b813b1 VG |
628 | policy->min = cppc_perf_to_khz(caps, caps->lowest_nonlinear_perf); |
629 | policy->max = cppc_perf_to_khz(caps, caps->nominal_perf); | |
73808d0f PP |
630 | |
631 | /* | |
632 | * Set cpuinfo.min_freq to Lowest to make the full range of performance | |
633 | * available if userspace wants to use any perf between lowest & lowest | |
634 | * nonlinear perf | |
635 | */ | |
50b813b1 VG |
636 | policy->cpuinfo.min_freq = cppc_perf_to_khz(caps, caps->lowest_perf); |
637 | policy->cpuinfo.max_freq = cppc_perf_to_khz(caps, caps->nominal_perf); | |
73808d0f | 638 | |
48ad8dc9 IV |
639 | policy->transition_delay_us = cppc_cpufreq_get_transition_delay_us(cpu); |
640 | policy->shared_type = cpu_data->shared_type; | |
5477fb3b | 641 | |
bf76bb20 IV |
642 | switch (policy->shared_type) { |
643 | case CPUFREQ_SHARED_TYPE_HW: | |
644 | case CPUFREQ_SHARED_TYPE_NONE: | |
645 | /* Nothing to be done - we'll have a policy for each CPU */ | |
646 | break; | |
647 | case CPUFREQ_SHARED_TYPE_ANY: | |
a28b2bfc IV |
648 | /* |
649 | * All CPUs in the domain will share a policy and all cpufreq | |
650 | * operations will use a single cppc_cpudata structure stored | |
651 | * in policy->driver_data. | |
652 | */ | |
48ad8dc9 | 653 | cpumask_copy(policy->cpus, cpu_data->shared_cpu_map); |
bf76bb20 IV |
654 | break; |
655 | default: | |
656 | pr_debug("Unsupported CPU co-ord type: %d\n", | |
657 | policy->shared_type); | |
fe2535a4 VK |
658 | ret = -EFAULT; |
659 | goto out; | |
5477fb3b AC |
660 | } |
661 | ||
3cc30dd0 | 662 | policy->fast_switch_possible = cppc_allow_fast_switch(); |
2d41dc23 | 663 | policy->dvfs_possible_from_any_cpu = true; |
3cc30dd0 | 664 | |
54e74df5 XW |
665 | /* |
666 | * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost | |
667 | * is supported. | |
668 | */ | |
bb025fb6 | 669 | if (caps->highest_perf > caps->nominal_perf) |
54e74df5 XW |
670 | boost_supported = true; |
671 | ||
5477fb3b | 672 | /* Set policy->cur to max now. The governors will adjust later. */ |
50b813b1 | 673 | policy->cur = cppc_perf_to_khz(caps, caps->highest_perf); |
bb025fb6 | 674 | cpu_data->perf_ctrls.desired_perf = caps->highest_perf; |
5477fb3b | 675 | |
48ad8dc9 | 676 | ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); |
fe2535a4 | 677 | if (ret) { |
5477fb3b | 678 | pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", |
bb025fb6 | 679 | caps->highest_perf, cpu, ret); |
fe2535a4 VK |
680 | goto out; |
681 | } | |
682 | ||
1eb5dde6 | 683 | cppc_cpufreq_cpu_fie_init(policy); |
fe2535a4 | 684 | return 0; |
5477fb3b | 685 | |
fe2535a4 VK |
686 | out: |
687 | cppc_cpufreq_put_cpu_data(policy); | |
5477fb3b AC |
688 | return ret; |
689 | } | |
690 | ||
9357a380 VK |
691 | static int cppc_cpufreq_cpu_exit(struct cpufreq_policy *policy) |
692 | { | |
693 | struct cppc_cpudata *cpu_data = policy->driver_data; | |
694 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; | |
695 | unsigned int cpu = policy->cpu; | |
696 | int ret; | |
697 | ||
1eb5dde6 VK |
698 | cppc_cpufreq_cpu_fie_exit(policy); |
699 | ||
9357a380 VK |
700 | cpu_data->perf_ctrls.desired_perf = caps->lowest_perf; |
701 | ||
702 | ret = cppc_set_perf(cpu, &cpu_data->perf_ctrls); | |
703 | if (ret) | |
704 | pr_debug("Err setting perf value:%d on CPU:%d. ret:%d\n", | |
705 | caps->lowest_perf, cpu, ret); | |
706 | ||
fe2535a4 | 707 | cppc_cpufreq_put_cpu_data(policy); |
9357a380 VK |
708 | return 0; |
709 | } | |
710 | ||
33477d84 GC |
711 | static inline u64 get_delta(u64 t1, u64 t0) |
712 | { | |
713 | if (t1 > t0 || t0 > ~(u32)0) | |
714 | return t1 - t0; | |
715 | ||
716 | return (u32)t1 - (u32)t0; | |
717 | } | |
718 | ||
1eb5dde6 VK |
719 | static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data, |
720 | struct cppc_perf_fb_ctrs *fb_ctrs_t0, | |
721 | struct cppc_perf_fb_ctrs *fb_ctrs_t1) | |
33477d84 GC |
722 | { |
723 | u64 delta_reference, delta_delivered; | |
1eb5dde6 | 724 | u64 reference_perf; |
33477d84 | 725 | |
eead1840 | 726 | reference_perf = fb_ctrs_t0->reference_perf; |
33477d84 | 727 | |
eead1840 VK |
728 | delta_reference = get_delta(fb_ctrs_t1->reference, |
729 | fb_ctrs_t0->reference); | |
730 | delta_delivered = get_delta(fb_ctrs_t1->delivered, | |
731 | fb_ctrs_t0->delivered); | |
33477d84 | 732 | |
1eb5dde6 VK |
733 | /* Check to avoid divide-by zero and invalid delivered_perf */ |
734 | if (!delta_reference || !delta_delivered) | |
735 | return cpu_data->perf_ctrls.desired_perf; | |
33477d84 | 736 | |
1eb5dde6 | 737 | return (reference_perf * delta_delivered) / delta_reference; |
33477d84 GC |
738 | } |
739 | ||
48ad8dc9 | 740 | static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) |
33477d84 GC |
741 | { |
742 | struct cppc_perf_fb_ctrs fb_ctrs_t0 = {0}, fb_ctrs_t1 = {0}; | |
a28b2bfc | 743 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
cf7de258 | 744 | struct cppc_cpudata *cpu_data; |
1eb5dde6 | 745 | u64 delivered_perf; |
33477d84 GC |
746 | int ret; |
747 | ||
cf7de258 AM |
748 | if (!policy) |
749 | return -ENODEV; | |
750 | ||
751 | cpu_data = policy->driver_data; | |
752 | ||
a28b2bfc IV |
753 | cpufreq_cpu_put(policy); |
754 | ||
48ad8dc9 | 755 | ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t0); |
33477d84 | 756 | if (ret) |
6a4fec4f | 757 | return 0; |
33477d84 GC |
758 | |
759 | udelay(2); /* 2usec delay between sampling */ | |
760 | ||
48ad8dc9 | 761 | ret = cppc_get_perf_ctrs(cpu, &fb_ctrs_t1); |
33477d84 | 762 | if (ret) |
6a4fec4f | 763 | return 0; |
33477d84 | 764 | |
1eb5dde6 VK |
765 | delivered_perf = cppc_perf_from_fbctrs(cpu_data, &fb_ctrs_t0, |
766 | &fb_ctrs_t1); | |
767 | ||
50b813b1 | 768 | return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); |
33477d84 GC |
769 | } |
770 | ||
54e74df5 XW |
771 | static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state) |
772 | { | |
a28b2bfc | 773 | struct cppc_cpudata *cpu_data = policy->driver_data; |
bb025fb6 | 774 | struct cppc_perf_caps *caps = &cpu_data->perf_caps; |
54e74df5 XW |
775 | int ret; |
776 | ||
777 | if (!boost_supported) { | |
778 | pr_err("BOOST not supported by CPU or firmware\n"); | |
779 | return -EINVAL; | |
780 | } | |
781 | ||
54e74df5 | 782 | if (state) |
50b813b1 | 783 | policy->max = cppc_perf_to_khz(caps, caps->highest_perf); |
54e74df5 | 784 | else |
50b813b1 | 785 | policy->max = cppc_perf_to_khz(caps, caps->nominal_perf); |
54e74df5 XW |
786 | policy->cpuinfo.max_freq = policy->max; |
787 | ||
788 | ret = freq_qos_update_request(policy->max_freq_req, policy->max); | |
789 | if (ret < 0) | |
790 | return ret; | |
791 | ||
792 | return 0; | |
793 | } | |
794 | ||
cfdc589f IV |
795 | static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) |
796 | { | |
a28b2bfc | 797 | struct cppc_cpudata *cpu_data = policy->driver_data; |
cfdc589f | 798 | |
a28b2bfc | 799 | return cpufreq_show_cpus(cpu_data->shared_cpu_map, buf); |
cfdc589f IV |
800 | } |
801 | cpufreq_freq_attr_ro(freqdomain_cpus); | |
802 | ||
803 | static struct freq_attr *cppc_cpufreq_attr[] = { | |
804 | &freqdomain_cpus, | |
805 | NULL, | |
806 | }; | |
807 | ||
5477fb3b AC |
808 | static struct cpufreq_driver cppc_cpufreq_driver = { |
809 | .flags = CPUFREQ_CONST_LOOPS, | |
810 | .verify = cppc_verify_policy, | |
811 | .target = cppc_cpufreq_set_target, | |
33477d84 | 812 | .get = cppc_cpufreq_get_rate, |
3cc30dd0 | 813 | .fast_switch = cppc_cpufreq_fast_switch, |
5477fb3b | 814 | .init = cppc_cpufreq_cpu_init, |
9357a380 | 815 | .exit = cppc_cpufreq_cpu_exit, |
54e74df5 | 816 | .set_boost = cppc_cpufreq_set_boost, |
cfdc589f | 817 | .attr = cppc_cpufreq_attr, |
5477fb3b AC |
818 | .name = "cppc_cpufreq", |
819 | }; | |
820 | ||
d88b0f0e VK |
821 | /* |
822 | * HISI platform does not support delivered performance counter and | |
823 | * reference performance counter. It can calculate the performance using the | |
824 | * platform specific mechanism. We reuse the desired performance register to | |
825 | * store the real performance calculated by the platform. | |
826 | */ | |
48ad8dc9 | 827 | static unsigned int hisi_cppc_cpufreq_get_rate(unsigned int cpu) |
d88b0f0e | 828 | { |
a28b2bfc | 829 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
cf7de258 | 830 | struct cppc_cpudata *cpu_data; |
d88b0f0e VK |
831 | u64 desired_perf; |
832 | int ret; | |
833 | ||
cf7de258 AM |
834 | if (!policy) |
835 | return -ENODEV; | |
836 | ||
837 | cpu_data = policy->driver_data; | |
838 | ||
a28b2bfc IV |
839 | cpufreq_cpu_put(policy); |
840 | ||
48ad8dc9 | 841 | ret = cppc_get_desired_perf(cpu, &desired_perf); |
d88b0f0e VK |
842 | if (ret < 0) |
843 | return -EIO; | |
844 | ||
50b813b1 | 845 | return cppc_perf_to_khz(&cpu_data->perf_caps, desired_perf); |
d88b0f0e VK |
846 | } |
847 | ||
848 | static void cppc_check_hisi_workaround(void) | |
849 | { | |
850 | struct acpi_table_header *tbl; | |
851 | acpi_status status = AE_OK; | |
852 | int i; | |
853 | ||
854 | status = acpi_get_table(ACPI_SIG_PCCT, 0, &tbl); | |
855 | if (ACPI_FAILURE(status) || !tbl) | |
856 | return; | |
857 | ||
858 | for (i = 0; i < ARRAY_SIZE(wa_info); i++) { | |
859 | if (!memcmp(wa_info[i].oem_id, tbl->oem_id, ACPI_OEM_ID_SIZE) && | |
860 | !memcmp(wa_info[i].oem_table_id, tbl->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) && | |
861 | wa_info[i].oem_revision == tbl->oem_revision) { | |
862 | /* Overwrite the get() callback */ | |
863 | cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate; | |
ae2df912 | 864 | fie_disabled = FIE_DISABLED; |
d88b0f0e VK |
865 | break; |
866 | } | |
867 | } | |
868 | ||
869 | acpi_put_table(tbl); | |
870 | } | |
871 | ||
5477fb3b AC |
872 | static int __init cppc_cpufreq_init(void) |
873 | { | |
1eb5dde6 VK |
874 | int ret; |
875 | ||
a2a9d185 | 876 | if (!acpi_cpc_valid()) |
5477fb3b AC |
877 | return -ENODEV; |
878 | ||
6c8d750f | 879 | cppc_check_hisi_workaround(); |
1eb5dde6 | 880 | cppc_freq_invariance_init(); |
d3c3db41 | 881 | populate_efficiency_class(); |
6c8d750f | 882 | |
1eb5dde6 VK |
883 | ret = cpufreq_register_driver(&cppc_cpufreq_driver); |
884 | if (ret) | |
885 | cppc_freq_invariance_exit(); | |
886 | ||
887 | return ret; | |
a28b2bfc | 888 | } |
5477fb3b | 889 | |
a28b2bfc IV |
890 | static inline void free_cpu_data(void) |
891 | { | |
892 | struct cppc_cpudata *iter, *tmp; | |
5477fb3b | 893 | |
a28b2bfc IV |
894 | list_for_each_entry_safe(iter, tmp, &cpu_data_list, node) { |
895 | free_cpumask_var(iter->shared_cpu_map); | |
896 | list_del(&iter->node); | |
897 | kfree(iter); | |
55b55abc | 898 | } |
5477fb3b | 899 | |
5477fb3b AC |
900 | } |
901 | ||
a29a1e76 AC |
902 | static void __exit cppc_cpufreq_exit(void) |
903 | { | |
a29a1e76 | 904 | cpufreq_unregister_driver(&cppc_cpufreq_driver); |
1eb5dde6 | 905 | cppc_freq_invariance_exit(); |
a29a1e76 | 906 | |
a28b2bfc | 907 | free_cpu_data(); |
a29a1e76 AC |
908 | } |
909 | ||
910 | module_exit(cppc_cpufreq_exit); | |
911 | MODULE_AUTHOR("Ashwin Chaugule"); | |
912 | MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); | |
913 | MODULE_LICENSE("GPL"); | |
914 | ||
5477fb3b | 915 | late_initcall(cppc_cpufreq_init); |
974f8649 | 916 | |
8ff3c226 | 917 | static const struct acpi_device_id cppc_acpi_ids[] __used = { |
974f8649 PP |
918 | {ACPI_PROCESSOR_DEVICE_HID, }, |
919 | {} | |
920 | }; | |
921 | ||
922 | MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids); |