Commit | Line | Data |
---|---|---|
ec437d71 HR |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* | |
3 | * amd-pstate.c - AMD Processor P-state Frequency Driver | |
4 | * | |
5 | * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. | |
6 | * | |
7 | * Author: Huang Rui <ray.huang@amd.com> | |
8 | * | |
9 | * AMD P-State introduces a new CPU performance scaling design for AMD | |
10 | * processors using the ACPI Collaborative Performance and Power Control (CPPC) | |
11 | * feature which works with the AMD SMU firmware providing a finer grained | |
12 | * frequency control range. It is to replace the legacy ACPI P-States control, | |
13 | * allows a flexible, low-latency interface for the Linux kernel to directly | |
14 | * communicate the performance hints to hardware. | |
15 | * | |
16 | * AMD P-State is supported on recent AMD Zen base CPU series include some of | |
17 | * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD | |
18 | * P-State supported system. And there are two types of hardware implementations | |
19 | * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. | |
20 | * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. | |
21 | */ | |
22 | ||
23 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
24 | ||
25 | #include <linux/kernel.h> | |
26 | #include <linux/module.h> | |
27 | #include <linux/init.h> | |
28 | #include <linux/smp.h> | |
29 | #include <linux/sched.h> | |
30 | #include <linux/cpufreq.h> | |
31 | #include <linux/compiler.h> | |
32 | #include <linux/dmi.h> | |
33 | #include <linux/slab.h> | |
34 | #include <linux/acpi.h> | |
35 | #include <linux/io.h> | |
36 | #include <linux/delay.h> | |
37 | #include <linux/uaccess.h> | |
38 | #include <linux/static_call.h> | |
f1375ec1 | 39 | #include <linux/amd-pstate.h> |
ec437d71 HR |
40 | |
41 | #include <acpi/processor.h> | |
42 | #include <acpi/cppc_acpi.h> | |
43 | ||
44 | #include <asm/msr.h> | |
45 | #include <asm/processor.h> | |
46 | #include <asm/cpufeature.h> | |
47 | #include <asm/cpu_device_id.h> | |
60e10f89 | 48 | #include "amd-pstate-trace.h" |
ec437d71 | 49 | |
ca08e46d PY |
50 | #define AMD_PSTATE_TRANSITION_LATENCY 20000 |
51 | #define AMD_PSTATE_TRANSITION_DELAY 1000 | |
ec437d71 | 52 | |
e059c184 HR |
53 | /* |
54 | * TODO: We need more time to fine tune processors with shared memory solution | |
55 | * with community together. | |
56 | * | |
57 | * There are some performance drops on the CPU benchmarks which reports from | |
58 | * Suse. We are co-working with them to fine tune the shared memory solution. So | |
59 | * we disable it by default to go acpi-cpufreq on these processors and add a | |
60 | * module parameter to be able to enable it manually for debugging. | |
61 | */ | |
ffa5096a | 62 | static struct cpufreq_driver *current_pstate_driver; |
ec437d71 | 63 | static struct cpufreq_driver amd_pstate_driver; |
ffa5096a | 64 | static struct cpufreq_driver amd_pstate_epp_driver; |
c88ad30e | 65 | static int cppc_state = AMD_PSTATE_UNDEFINED; |
217e6778 | 66 | static bool cppc_enabled; |
36c5014e | 67 | |
ffa5096a PY |
68 | /* |
69 | * AMD Energy Preference Performance (EPP) | |
70 | * The EPP is used in the CCLK DPM controller to drive | |
71 | * the frequency that a core is going to operate during | |
72 | * short periods of activity. EPP values will be utilized for | |
73 | * different OS profiles (balanced, performance, power savings) | |
74 | * display strings corresponding to EPP index in the | |
75 | * energy_perf_strings[] | |
76 | * index String | |
77 | *------------------------------------- | |
78 | * 0 default | |
79 | * 1 performance | |
80 | * 2 balance_performance | |
81 | * 3 balance_power | |
82 | * 4 power | |
83 | */ | |
84 | enum energy_perf_value_index { | |
85 | EPP_INDEX_DEFAULT = 0, | |
86 | EPP_INDEX_PERFORMANCE, | |
87 | EPP_INDEX_BALANCE_PERFORMANCE, | |
88 | EPP_INDEX_BALANCE_POWERSAVE, | |
89 | EPP_INDEX_POWERSAVE, | |
90 | }; | |
91 | ||
92 | static const char * const energy_perf_strings[] = { | |
93 | [EPP_INDEX_DEFAULT] = "default", | |
94 | [EPP_INDEX_PERFORMANCE] = "performance", | |
95 | [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", | |
96 | [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", | |
97 | [EPP_INDEX_POWERSAVE] = "power", | |
98 | NULL | |
99 | }; | |
100 | ||
101 | static unsigned int epp_values[] = { | |
102 | [EPP_INDEX_DEFAULT] = 0, | |
103 | [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, | |
104 | [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, | |
105 | [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, | |
106 | [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, | |
107 | }; | |
108 | ||
3ca7bc81 WK |
109 | typedef int (*cppc_mode_transition_fn)(int); |
110 | ||
36c5014e WK |
111 | static inline int get_mode_idx_from_str(const char *str, size_t size) |
112 | { | |
113 | int i; | |
114 | ||
115 | for (i=0; i < AMD_PSTATE_MAX; i++) { | |
116 | if (!strncmp(str, amd_pstate_mode_string[i], size)) | |
117 | return i; | |
118 | } | |
119 | return -EINVAL; | |
120 | } | |
ec437d71 | 121 | |
ffa5096a PY |
122 | static DEFINE_MUTEX(amd_pstate_limits_lock); |
123 | static DEFINE_MUTEX(amd_pstate_driver_lock); | |
124 | ||
125 | static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) | |
126 | { | |
127 | u64 epp; | |
128 | int ret; | |
129 | ||
130 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
131 | if (!cppc_req_cached) { | |
132 | epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, | |
133 | &cppc_req_cached); | |
134 | if (epp) | |
135 | return epp; | |
136 | } | |
137 | epp = (cppc_req_cached >> 24) & 0xFF; | |
138 | } else { | |
139 | ret = cppc_get_epp_perf(cpudata->cpu, &epp); | |
140 | if (ret < 0) { | |
141 | pr_debug("Could not retrieve energy perf value (%d)\n", ret); | |
142 | return -EIO; | |
143 | } | |
144 | } | |
145 | ||
146 | return (s16)(epp & 0xff); | |
147 | } | |
148 | ||
149 | static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) | |
150 | { | |
151 | s16 epp; | |
152 | int index = -EINVAL; | |
153 | ||
154 | epp = amd_pstate_get_epp(cpudata, 0); | |
155 | if (epp < 0) | |
156 | return epp; | |
157 | ||
158 | switch (epp) { | |
159 | case AMD_CPPC_EPP_PERFORMANCE: | |
160 | index = EPP_INDEX_PERFORMANCE; | |
161 | break; | |
162 | case AMD_CPPC_EPP_BALANCE_PERFORMANCE: | |
163 | index = EPP_INDEX_BALANCE_PERFORMANCE; | |
164 | break; | |
165 | case AMD_CPPC_EPP_BALANCE_POWERSAVE: | |
166 | index = EPP_INDEX_BALANCE_POWERSAVE; | |
167 | break; | |
168 | case AMD_CPPC_EPP_POWERSAVE: | |
169 | index = EPP_INDEX_POWERSAVE; | |
170 | break; | |
171 | default: | |
172 | break; | |
173 | } | |
174 | ||
175 | return index; | |
176 | } | |
177 | ||
178 | static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) | |
179 | { | |
180 | int ret; | |
181 | struct cppc_perf_ctrls perf_ctrls; | |
182 | ||
183 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
184 | u64 value = READ_ONCE(cpudata->cppc_req_cached); | |
185 | ||
186 | value &= ~GENMASK_ULL(31, 24); | |
187 | value |= (u64)epp << 24; | |
188 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
189 | ||
190 | ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
191 | if (!ret) | |
192 | cpudata->epp_cached = epp; | |
193 | } else { | |
194 | perf_ctrls.energy_perf = epp; | |
195 | ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); | |
196 | if (ret) { | |
197 | pr_debug("failed to set energy perf value (%d)\n", ret); | |
198 | return ret; | |
199 | } | |
200 | cpudata->epp_cached = epp; | |
201 | } | |
202 | ||
203 | return ret; | |
204 | } | |
205 | ||
206 | static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, | |
207 | int pref_index) | |
208 | { | |
209 | int epp = -EINVAL; | |
210 | int ret; | |
211 | ||
212 | if (!pref_index) { | |
213 | pr_debug("EPP pref_index is invalid\n"); | |
214 | return -EINVAL; | |
215 | } | |
216 | ||
217 | if (epp == -EINVAL) | |
218 | epp = epp_values[pref_index]; | |
219 | ||
220 | if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { | |
221 | pr_debug("EPP cannot be set under performance policy\n"); | |
222 | return -EBUSY; | |
223 | } | |
224 | ||
225 | ret = amd_pstate_set_epp(cpudata, epp); | |
226 | ||
227 | return ret; | |
228 | } | |
229 | ||
e059c184 | 230 | static inline int pstate_enable(bool enable) |
ec437d71 | 231 | { |
217e6778 WK |
232 | int ret, cpu; |
233 | unsigned long logical_proc_id_mask = 0; | |
234 | ||
235 | if (enable == cppc_enabled) | |
236 | return 0; | |
237 | ||
238 | for_each_present_cpu(cpu) { | |
239 | unsigned long logical_id = topology_logical_die_id(cpu); | |
240 | ||
241 | if (test_bit(logical_id, &logical_proc_id_mask)) | |
242 | continue; | |
243 | ||
244 | set_bit(logical_id, &logical_proc_id_mask); | |
245 | ||
246 | ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE, | |
247 | enable); | |
248 | if (ret) | |
249 | return ret; | |
250 | } | |
251 | ||
252 | cppc_enabled = enable; | |
253 | return 0; | |
ec437d71 HR |
254 | } |
255 | ||
e059c184 HR |
256 | static int cppc_enable(bool enable) |
257 | { | |
258 | int cpu, ret = 0; | |
ffa5096a | 259 | struct cppc_perf_ctrls perf_ctrls; |
e059c184 | 260 | |
217e6778 WK |
261 | if (enable == cppc_enabled) |
262 | return 0; | |
263 | ||
e059c184 HR |
264 | for_each_present_cpu(cpu) { |
265 | ret = cppc_set_enable(cpu, enable); | |
266 | if (ret) | |
267 | return ret; | |
ffa5096a PY |
268 | |
269 | /* Enable autonomous mode for EPP */ | |
270 | if (cppc_state == AMD_PSTATE_ACTIVE) { | |
271 | /* Set desired perf as zero to allow EPP firmware control */ | |
272 | perf_ctrls.desired_perf = 0; | |
273 | ret = cppc_set_perf(cpu, &perf_ctrls); | |
274 | if (ret) | |
275 | return ret; | |
276 | } | |
e059c184 HR |
277 | } |
278 | ||
217e6778 | 279 | cppc_enabled = enable; |
e059c184 HR |
280 | return ret; |
281 | } | |
282 | ||
283 | DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); | |
284 | ||
285 | static inline int amd_pstate_enable(bool enable) | |
286 | { | |
287 | return static_call(amd_pstate_enable)(enable); | |
288 | } | |
289 | ||
290 | static int pstate_init_perf(struct amd_cpudata *cpudata) | |
ec437d71 HR |
291 | { |
292 | u64 cap1; | |
bedadcfb | 293 | u32 highest_perf; |
ec437d71 HR |
294 | |
295 | int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, | |
296 | &cap1); | |
297 | if (ret) | |
298 | return ret; | |
299 | ||
300 | /* | |
301 | * TODO: Introduce AMD specific power feature. | |
302 | * | |
303 | * CPPC entry doesn't indicate the highest performance in some ASICs. | |
304 | */ | |
bedadcfb PY |
305 | highest_perf = amd_get_highest_perf(); |
306 | if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) | |
307 | highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); | |
308 | ||
309 | WRITE_ONCE(cpudata->highest_perf, highest_perf); | |
ec437d71 HR |
310 | |
311 | WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); | |
312 | WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); | |
313 | WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); | |
314 | ||
315 | return 0; | |
316 | } | |
317 | ||
e059c184 HR |
318 | static int cppc_init_perf(struct amd_cpudata *cpudata) |
319 | { | |
320 | struct cppc_perf_caps cppc_perf; | |
bedadcfb | 321 | u32 highest_perf; |
e059c184 HR |
322 | |
323 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
324 | if (ret) | |
325 | return ret; | |
326 | ||
bedadcfb PY |
327 | highest_perf = amd_get_highest_perf(); |
328 | if (highest_perf > cppc_perf.highest_perf) | |
329 | highest_perf = cppc_perf.highest_perf; | |
330 | ||
331 | WRITE_ONCE(cpudata->highest_perf, highest_perf); | |
e059c184 HR |
332 | |
333 | WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); | |
334 | WRITE_ONCE(cpudata->lowest_nonlinear_perf, | |
335 | cppc_perf.lowest_nonlinear_perf); | |
336 | WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); | |
337 | ||
2dd6d0eb WK |
338 | if (cppc_state == AMD_PSTATE_ACTIVE) |
339 | return 0; | |
340 | ||
341 | ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); | |
342 | if (ret) { | |
343 | pr_warn("failed to get auto_sel, ret: %d\n", ret); | |
344 | return 0; | |
345 | } | |
346 | ||
347 | ret = cppc_set_auto_sel(cpudata->cpu, | |
348 | (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); | |
349 | ||
350 | if (ret) | |
351 | pr_warn("failed to set auto_sel, ret: %d\n", ret); | |
352 | ||
353 | return ret; | |
e059c184 HR |
354 | } |
355 | ||
356 | DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); | |
357 | ||
358 | static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) | |
359 | { | |
360 | return static_call(amd_pstate_init_perf)(cpudata); | |
361 | } | |
362 | ||
363 | static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, | |
364 | u32 des_perf, u32 max_perf, bool fast_switch) | |
ec437d71 HR |
365 | { |
366 | if (fast_switch) | |
367 | wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); | |
368 | else | |
369 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, | |
370 | READ_ONCE(cpudata->cppc_req_cached)); | |
371 | } | |
372 | ||
e059c184 HR |
373 | static void cppc_update_perf(struct amd_cpudata *cpudata, |
374 | u32 min_perf, u32 des_perf, | |
375 | u32 max_perf, bool fast_switch) | |
376 | { | |
377 | struct cppc_perf_ctrls perf_ctrls; | |
378 | ||
379 | perf_ctrls.max_perf = max_perf; | |
380 | perf_ctrls.min_perf = min_perf; | |
381 | perf_ctrls.desired_perf = des_perf; | |
382 | ||
383 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
384 | } | |
385 | ||
386 | DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); | |
387 | ||
388 | static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, | |
389 | u32 min_perf, u32 des_perf, | |
390 | u32 max_perf, bool fast_switch) | |
391 | { | |
392 | static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, | |
393 | max_perf, fast_switch); | |
394 | } | |
395 | ||
23c296fb JS |
396 | static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) |
397 | { | |
398 | u64 aperf, mperf, tsc; | |
399 | unsigned long flags; | |
400 | ||
401 | local_irq_save(flags); | |
402 | rdmsrl(MSR_IA32_APERF, aperf); | |
403 | rdmsrl(MSR_IA32_MPERF, mperf); | |
404 | tsc = rdtsc(); | |
405 | ||
406 | if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { | |
407 | local_irq_restore(flags); | |
408 | return false; | |
409 | } | |
410 | ||
411 | local_irq_restore(flags); | |
412 | ||
413 | cpudata->cur.aperf = aperf; | |
414 | cpudata->cur.mperf = mperf; | |
415 | cpudata->cur.tsc = tsc; | |
416 | cpudata->cur.aperf -= cpudata->prev.aperf; | |
417 | cpudata->cur.mperf -= cpudata->prev.mperf; | |
418 | cpudata->cur.tsc -= cpudata->prev.tsc; | |
419 | ||
420 | cpudata->prev.aperf = aperf; | |
421 | cpudata->prev.mperf = mperf; | |
422 | cpudata->prev.tsc = tsc; | |
423 | ||
424 | cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); | |
425 | ||
426 | return true; | |
427 | } | |
428 | ||
ec437d71 | 429 | static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, |
2dd6d0eb | 430 | u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) |
ec437d71 HR |
431 | { |
432 | u64 prev = READ_ONCE(cpudata->cppc_req_cached); | |
433 | u64 value = prev; | |
434 | ||
0e9a8638 | 435 | des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); |
2dd6d0eb WK |
436 | |
437 | if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { | |
438 | min_perf = des_perf; | |
439 | des_perf = 0; | |
440 | } | |
441 | ||
ec437d71 HR |
442 | value &= ~AMD_CPPC_MIN_PERF(~0L); |
443 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
444 | ||
445 | value &= ~AMD_CPPC_DES_PERF(~0L); | |
446 | value |= AMD_CPPC_DES_PERF(des_perf); | |
447 | ||
448 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
449 | value |= AMD_CPPC_MAX_PERF(max_perf); | |
450 | ||
23c296fb JS |
451 | if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { |
452 | trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, | |
453 | cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, | |
454 | cpudata->cpu, (value != prev), fast_switch); | |
455 | } | |
60e10f89 | 456 | |
ec437d71 HR |
457 | if (value == prev) |
458 | return; | |
459 | ||
460 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
461 | ||
462 | amd_pstate_update_perf(cpudata, min_perf, des_perf, | |
463 | max_perf, fast_switch); | |
464 | } | |
465 | ||
466 | static int amd_pstate_verify(struct cpufreq_policy_data *policy) | |
467 | { | |
468 | cpufreq_verify_within_cpu_limits(policy); | |
469 | ||
470 | return 0; | |
471 | } | |
472 | ||
4badf2eb GS |
473 | static int amd_pstate_update_freq(struct cpufreq_policy *policy, |
474 | unsigned int target_freq, bool fast_switch) | |
ec437d71 HR |
475 | { |
476 | struct cpufreq_freqs freqs; | |
477 | struct amd_cpudata *cpudata = policy->driver_data; | |
478 | unsigned long max_perf, min_perf, des_perf, cap_perf; | |
479 | ||
480 | if (!cpudata->max_freq) | |
481 | return -ENODEV; | |
482 | ||
483 | cap_perf = READ_ONCE(cpudata->highest_perf); | |
b185c505 | 484 | min_perf = READ_ONCE(cpudata->lowest_perf); |
ec437d71 HR |
485 | max_perf = cap_perf; |
486 | ||
487 | freqs.old = policy->cur; | |
488 | freqs.new = target_freq; | |
489 | ||
490 | des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, | |
491 | cpudata->max_freq); | |
492 | ||
4badf2eb GS |
493 | WARN_ON(fast_switch && !policy->fast_switch_enabled); |
494 | /* | |
495 | * If fast_switch is desired, then there aren't any registered | |
496 | * transition notifiers. See comment for | |
497 | * cpufreq_enable_fast_switch(). | |
498 | */ | |
499 | if (!fast_switch) | |
500 | cpufreq_freq_transition_begin(policy, &freqs); | |
501 | ||
ec437d71 | 502 | amd_pstate_update(cpudata, min_perf, des_perf, |
4badf2eb GS |
503 | max_perf, fast_switch, policy->governor->flags); |
504 | ||
505 | if (!fast_switch) | |
506 | cpufreq_freq_transition_end(policy, &freqs, false); | |
ec437d71 HR |
507 | |
508 | return 0; | |
509 | } | |
510 | ||
4badf2eb GS |
511 | static int amd_pstate_target(struct cpufreq_policy *policy, |
512 | unsigned int target_freq, | |
513 | unsigned int relation) | |
514 | { | |
515 | return amd_pstate_update_freq(policy, target_freq, false); | |
516 | } | |
517 | ||
518 | static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, | |
519 | unsigned int target_freq) | |
520 | { | |
521 | return amd_pstate_update_freq(policy, target_freq, true); | |
522 | } | |
523 | ||
1d215f03 HR |
524 | static void amd_pstate_adjust_perf(unsigned int cpu, |
525 | unsigned long _min_perf, | |
526 | unsigned long target_perf, | |
527 | unsigned long capacity) | |
528 | { | |
529 | unsigned long max_perf, min_perf, des_perf, | |
3bf8c630 | 530 | cap_perf, lowest_nonlinear_perf, max_freq; |
1d215f03 HR |
531 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
532 | struct amd_cpudata *cpudata = policy->driver_data; | |
3bf8c630 | 533 | unsigned int target_freq; |
1d215f03 HR |
534 | |
535 | cap_perf = READ_ONCE(cpudata->highest_perf); | |
536 | lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); | |
3bf8c630 | 537 | max_freq = READ_ONCE(cpudata->max_freq); |
1d215f03 HR |
538 | |
539 | des_perf = cap_perf; | |
540 | if (target_perf < capacity) | |
541 | des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); | |
542 | ||
543 | min_perf = READ_ONCE(cpudata->highest_perf); | |
544 | if (_min_perf < capacity) | |
545 | min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); | |
546 | ||
547 | if (min_perf < lowest_nonlinear_perf) | |
548 | min_perf = lowest_nonlinear_perf; | |
549 | ||
550 | max_perf = cap_perf; | |
551 | if (max_perf < min_perf) | |
552 | max_perf = min_perf; | |
553 | ||
3bf8c630 WK |
554 | des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); |
555 | target_freq = div_u64(des_perf * max_freq, max_perf); | |
556 | policy->cur = target_freq; | |
557 | ||
2dd6d0eb WK |
558 | amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, |
559 | policy->governor->flags); | |
4f3085f8 | 560 | cpufreq_cpu_put(policy); |
1d215f03 HR |
561 | } |
562 | ||
ec437d71 HR |
563 | static int amd_get_min_freq(struct amd_cpudata *cpudata) |
564 | { | |
565 | struct cppc_perf_caps cppc_perf; | |
566 | ||
567 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
568 | if (ret) | |
569 | return ret; | |
570 | ||
571 | /* Switch to khz */ | |
572 | return cppc_perf.lowest_freq * 1000; | |
573 | } | |
574 | ||
575 | static int amd_get_max_freq(struct amd_cpudata *cpudata) | |
576 | { | |
577 | struct cppc_perf_caps cppc_perf; | |
578 | u32 max_perf, max_freq, nominal_freq, nominal_perf; | |
579 | u64 boost_ratio; | |
580 | ||
581 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
582 | if (ret) | |
583 | return ret; | |
584 | ||
585 | nominal_freq = cppc_perf.nominal_freq; | |
586 | nominal_perf = READ_ONCE(cpudata->nominal_perf); | |
587 | max_perf = READ_ONCE(cpudata->highest_perf); | |
588 | ||
589 | boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, | |
590 | nominal_perf); | |
591 | ||
592 | max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; | |
593 | ||
594 | /* Switch to khz */ | |
595 | return max_freq * 1000; | |
596 | } | |
597 | ||
598 | static int amd_get_nominal_freq(struct amd_cpudata *cpudata) | |
599 | { | |
600 | struct cppc_perf_caps cppc_perf; | |
601 | ||
602 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
603 | if (ret) | |
604 | return ret; | |
605 | ||
606 | /* Switch to khz */ | |
607 | return cppc_perf.nominal_freq * 1000; | |
608 | } | |
609 | ||
610 | static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) | |
611 | { | |
612 | struct cppc_perf_caps cppc_perf; | |
613 | u32 lowest_nonlinear_freq, lowest_nonlinear_perf, | |
614 | nominal_freq, nominal_perf; | |
615 | u64 lowest_nonlinear_ratio; | |
616 | ||
617 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
618 | if (ret) | |
619 | return ret; | |
620 | ||
621 | nominal_freq = cppc_perf.nominal_freq; | |
622 | nominal_perf = READ_ONCE(cpudata->nominal_perf); | |
623 | ||
624 | lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; | |
625 | ||
626 | lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, | |
627 | nominal_perf); | |
628 | ||
629 | lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; | |
630 | ||
631 | /* Switch to khz */ | |
632 | return lowest_nonlinear_freq * 1000; | |
633 | } | |
634 | ||
41271016 HR |
635 | static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) |
636 | { | |
637 | struct amd_cpudata *cpudata = policy->driver_data; | |
638 | int ret; | |
639 | ||
640 | if (!cpudata->boost_supported) { | |
641 | pr_err("Boost mode is not supported by this processor or SBIOS\n"); | |
642 | return -EINVAL; | |
643 | } | |
644 | ||
645 | if (state) | |
646 | policy->cpuinfo.max_freq = cpudata->max_freq; | |
647 | else | |
648 | policy->cpuinfo.max_freq = cpudata->nominal_freq; | |
649 | ||
650 | policy->max = policy->cpuinfo.max_freq; | |
651 | ||
652 | ret = freq_qos_update_request(&cpudata->req[1], | |
653 | policy->cpuinfo.max_freq); | |
654 | if (ret < 0) | |
655 | return ret; | |
656 | ||
657 | return 0; | |
658 | } | |
659 | ||
660 | static void amd_pstate_boost_init(struct amd_cpudata *cpudata) | |
661 | { | |
662 | u32 highest_perf, nominal_perf; | |
663 | ||
664 | highest_perf = READ_ONCE(cpudata->highest_perf); | |
665 | nominal_perf = READ_ONCE(cpudata->nominal_perf); | |
666 | ||
667 | if (highest_perf <= nominal_perf) | |
668 | return; | |
669 | ||
670 | cpudata->boost_supported = true; | |
ffa5096a | 671 | current_pstate_driver->boost_enabled = true; |
41271016 HR |
672 | } |
673 | ||
919f4557 WK |
674 | static void amd_perf_ctl_reset(unsigned int cpu) |
675 | { | |
676 | wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); | |
677 | } | |
678 | ||
ec437d71 HR |
679 | static int amd_pstate_cpu_init(struct cpufreq_policy *policy) |
680 | { | |
681 | int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; | |
682 | struct device *dev; | |
683 | struct amd_cpudata *cpudata; | |
684 | ||
919f4557 WK |
685 | /* |
686 | * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, | |
687 | * which is ideal for initialization process. | |
688 | */ | |
689 | amd_perf_ctl_reset(policy->cpu); | |
ec437d71 HR |
690 | dev = get_cpu_device(policy->cpu); |
691 | if (!dev) | |
692 | return -ENODEV; | |
693 | ||
694 | cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); | |
695 | if (!cpudata) | |
696 | return -ENOMEM; | |
697 | ||
698 | cpudata->cpu = policy->cpu; | |
699 | ||
700 | ret = amd_pstate_init_perf(cpudata); | |
701 | if (ret) | |
41271016 | 702 | goto free_cpudata1; |
ec437d71 HR |
703 | |
704 | min_freq = amd_get_min_freq(cpudata); | |
705 | max_freq = amd_get_max_freq(cpudata); | |
706 | nominal_freq = amd_get_nominal_freq(cpudata); | |
707 | lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); | |
708 | ||
709 | if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { | |
710 | dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", | |
711 | min_freq, max_freq); | |
712 | ret = -EINVAL; | |
41271016 | 713 | goto free_cpudata1; |
ec437d71 HR |
714 | } |
715 | ||
716 | policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; | |
717 | policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; | |
718 | ||
719 | policy->min = min_freq; | |
720 | policy->max = max_freq; | |
721 | ||
722 | policy->cpuinfo.min_freq = min_freq; | |
723 | policy->cpuinfo.max_freq = max_freq; | |
724 | ||
725 | /* It will be updated by governor */ | |
726 | policy->cur = policy->cpuinfo.min_freq; | |
727 | ||
e059c184 HR |
728 | if (boot_cpu_has(X86_FEATURE_CPPC)) |
729 | policy->fast_switch_possible = true; | |
1d215f03 | 730 | |
41271016 HR |
731 | ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], |
732 | FREQ_QOS_MIN, policy->cpuinfo.min_freq); | |
733 | if (ret < 0) { | |
734 | dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); | |
735 | goto free_cpudata1; | |
736 | } | |
737 | ||
738 | ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], | |
739 | FREQ_QOS_MAX, policy->cpuinfo.max_freq); | |
740 | if (ret < 0) { | |
741 | dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); | |
742 | goto free_cpudata2; | |
743 | } | |
744 | ||
ec437d71 HR |
745 | /* Initial processor data capability frequencies */ |
746 | cpudata->max_freq = max_freq; | |
747 | cpudata->min_freq = min_freq; | |
748 | cpudata->nominal_freq = nominal_freq; | |
749 | cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; | |
750 | ||
751 | policy->driver_data = cpudata; | |
752 | ||
41271016 | 753 | amd_pstate_boost_init(cpudata); |
abd61c08 PY |
754 | if (!current_pstate_driver->adjust_perf) |
755 | current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; | |
41271016 | 756 | |
ec437d71 HR |
757 | return 0; |
758 | ||
41271016 HR |
759 | free_cpudata2: |
760 | freq_qos_remove_request(&cpudata->req[0]); | |
761 | free_cpudata1: | |
ec437d71 HR |
762 | kfree(cpudata); |
763 | return ret; | |
764 | } | |
765 | ||
766 | static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) | |
767 | { | |
4f59540c | 768 | struct amd_cpudata *cpudata = policy->driver_data; |
ec437d71 | 769 | |
41271016 HR |
770 | freq_qos_remove_request(&cpudata->req[1]); |
771 | freq_qos_remove_request(&cpudata->req[0]); | |
4badf2eb | 772 | policy->fast_switch_possible = false; |
ec437d71 HR |
773 | kfree(cpudata); |
774 | ||
775 | return 0; | |
776 | } | |
777 | ||
b376471f JS |
778 | static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) |
779 | { | |
780 | int ret; | |
781 | ||
782 | ret = amd_pstate_enable(true); | |
783 | if (ret) | |
784 | pr_err("failed to enable amd-pstate during resume, return %d\n", ret); | |
785 | ||
786 | return ret; | |
787 | } | |
788 | ||
789 | static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) | |
790 | { | |
791 | int ret; | |
792 | ||
793 | ret = amd_pstate_enable(false); | |
794 | if (ret) | |
795 | pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); | |
796 | ||
797 | return ret; | |
798 | } | |
799 | ||
ec4e3326 HR |
800 | /* Sysfs attributes */ |
801 | ||
802 | /* | |
803 | * This frequency is to indicate the maximum hardware frequency. | |
804 | * If boost is not active but supported, the frequency will be larger than the | |
805 | * one in cpuinfo. | |
806 | */ | |
807 | static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, | |
808 | char *buf) | |
809 | { | |
810 | int max_freq; | |
4f59540c | 811 | struct amd_cpudata *cpudata = policy->driver_data; |
ec4e3326 HR |
812 | |
813 | max_freq = amd_get_max_freq(cpudata); | |
814 | if (max_freq < 0) | |
815 | return max_freq; | |
816 | ||
3ec32b6d | 817 | return sysfs_emit(buf, "%u\n", max_freq); |
ec4e3326 HR |
818 | } |
819 | ||
820 | static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, | |
821 | char *buf) | |
822 | { | |
823 | int freq; | |
4f59540c | 824 | struct amd_cpudata *cpudata = policy->driver_data; |
ec4e3326 HR |
825 | |
826 | freq = amd_get_lowest_nonlinear_freq(cpudata); | |
827 | if (freq < 0) | |
828 | return freq; | |
829 | ||
3ec32b6d | 830 | return sysfs_emit(buf, "%u\n", freq); |
ec4e3326 HR |
831 | } |
832 | ||
3ad7fde1 HR |
833 | /* |
834 | * In some of ASICs, the highest_perf is not the one in the _CPC table, so we | |
835 | * need to expose it to sysfs. | |
836 | */ | |
837 | static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, | |
838 | char *buf) | |
839 | { | |
840 | u32 perf; | |
841 | struct amd_cpudata *cpudata = policy->driver_data; | |
842 | ||
843 | perf = READ_ONCE(cpudata->highest_perf); | |
844 | ||
3ec32b6d | 845 | return sysfs_emit(buf, "%u\n", perf); |
3ad7fde1 HR |
846 | } |
847 | ||
ffa5096a PY |
848 | static ssize_t show_energy_performance_available_preferences( |
849 | struct cpufreq_policy *policy, char *buf) | |
850 | { | |
851 | int i = 0; | |
852 | int offset = 0; | |
853 | ||
854 | while (energy_perf_strings[i] != NULL) | |
855 | offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); | |
856 | ||
857 | sysfs_emit_at(buf, offset, "\n"); | |
858 | ||
859 | return offset; | |
860 | } | |
861 | ||
862 | static ssize_t store_energy_performance_preference( | |
863 | struct cpufreq_policy *policy, const char *buf, size_t count) | |
864 | { | |
865 | struct amd_cpudata *cpudata = policy->driver_data; | |
866 | char str_preference[21]; | |
867 | ssize_t ret; | |
868 | ||
869 | ret = sscanf(buf, "%20s", str_preference); | |
870 | if (ret != 1) | |
871 | return -EINVAL; | |
872 | ||
873 | ret = match_string(energy_perf_strings, -1, str_preference); | |
874 | if (ret < 0) | |
875 | return -EINVAL; | |
876 | ||
877 | mutex_lock(&amd_pstate_limits_lock); | |
878 | ret = amd_pstate_set_energy_pref_index(cpudata, ret); | |
879 | mutex_unlock(&amd_pstate_limits_lock); | |
880 | ||
881 | return ret ?: count; | |
882 | } | |
883 | ||
884 | static ssize_t show_energy_performance_preference( | |
885 | struct cpufreq_policy *policy, char *buf) | |
886 | { | |
887 | struct amd_cpudata *cpudata = policy->driver_data; | |
888 | int preference; | |
889 | ||
890 | preference = amd_pstate_get_energy_pref_index(cpudata); | |
891 | if (preference < 0) | |
892 | return preference; | |
893 | ||
894 | return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); | |
895 | } | |
896 | ||
3ca7bc81 WK |
897 | static void amd_pstate_driver_cleanup(void) |
898 | { | |
899 | amd_pstate_enable(false); | |
900 | cppc_state = AMD_PSTATE_DISABLE; | |
901 | current_pstate_driver = NULL; | |
902 | } | |
903 | ||
904 | static int amd_pstate_register_driver(int mode) | |
905 | { | |
906 | int ret; | |
907 | ||
908 | if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) | |
909 | current_pstate_driver = &amd_pstate_driver; | |
910 | else if (mode == AMD_PSTATE_ACTIVE) | |
911 | current_pstate_driver = &amd_pstate_epp_driver; | |
912 | else | |
913 | return -EINVAL; | |
914 | ||
915 | cppc_state = mode; | |
916 | ret = cpufreq_register_driver(current_pstate_driver); | |
917 | if (ret) { | |
918 | amd_pstate_driver_cleanup(); | |
919 | return ret; | |
920 | } | |
921 | return 0; | |
922 | } | |
923 | ||
924 | static int amd_pstate_unregister_driver(int dummy) | |
925 | { | |
926 | cpufreq_unregister_driver(current_pstate_driver); | |
927 | amd_pstate_driver_cleanup(); | |
928 | return 0; | |
929 | } | |
930 | ||
931 | static int amd_pstate_change_mode_without_dvr_change(int mode) | |
932 | { | |
933 | int cpu = 0; | |
934 | ||
935 | cppc_state = mode; | |
936 | ||
937 | if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) | |
938 | return 0; | |
939 | ||
940 | for_each_present_cpu(cpu) { | |
941 | cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); | |
942 | } | |
943 | ||
944 | return 0; | |
945 | } | |
946 | ||
947 | static int amd_pstate_change_driver_mode(int mode) | |
948 | { | |
949 | int ret; | |
950 | ||
951 | ret = amd_pstate_unregister_driver(0); | |
952 | if (ret) | |
953 | return ret; | |
954 | ||
955 | ret = amd_pstate_register_driver(mode); | |
956 | if (ret) | |
957 | return ret; | |
958 | ||
959 | return 0; | |
960 | } | |
961 | ||
11fa52fe | 962 | static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { |
3ca7bc81 WK |
963 | [AMD_PSTATE_DISABLE] = { |
964 | [AMD_PSTATE_DISABLE] = NULL, | |
965 | [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, | |
966 | [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, | |
967 | [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, | |
968 | }, | |
969 | [AMD_PSTATE_PASSIVE] = { | |
970 | [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, | |
971 | [AMD_PSTATE_PASSIVE] = NULL, | |
972 | [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, | |
973 | [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, | |
974 | }, | |
975 | [AMD_PSTATE_ACTIVE] = { | |
976 | [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, | |
977 | [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, | |
978 | [AMD_PSTATE_ACTIVE] = NULL, | |
979 | [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, | |
980 | }, | |
981 | [AMD_PSTATE_GUIDED] = { | |
982 | [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, | |
983 | [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, | |
984 | [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, | |
985 | [AMD_PSTATE_GUIDED] = NULL, | |
986 | }, | |
987 | }; | |
988 | ||
abd61c08 PY |
989 | static ssize_t amd_pstate_show_status(char *buf) |
990 | { | |
991 | if (!current_pstate_driver) | |
992 | return sysfs_emit(buf, "disable\n"); | |
993 | ||
994 | return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); | |
995 | } | |
996 | ||
abd61c08 PY |
997 | static int amd_pstate_update_status(const char *buf, size_t size) |
998 | { | |
abd61c08 PY |
999 | int mode_idx; |
1000 | ||
3ca7bc81 | 1001 | if (size > strlen("passive") || size < strlen("active")) |
abd61c08 | 1002 | return -EINVAL; |
abd61c08 | 1003 | |
3ca7bc81 | 1004 | mode_idx = get_mode_idx_from_str(buf, size); |
abd61c08 | 1005 | |
3ca7bc81 WK |
1006 | if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) |
1007 | return -EINVAL; | |
abd61c08 | 1008 | |
3ca7bc81 WK |
1009 | if (mode_state_machine[cppc_state][mode_idx]) |
1010 | return mode_state_machine[cppc_state][mode_idx](mode_idx); | |
abd61c08 | 1011 | |
3ca7bc81 | 1012 | return 0; |
abd61c08 PY |
1013 | } |
1014 | ||
5e720f8c TW |
1015 | static ssize_t status_show(struct device *dev, |
1016 | struct device_attribute *attr, char *buf) | |
abd61c08 PY |
1017 | { |
1018 | ssize_t ret; | |
1019 | ||
1020 | mutex_lock(&amd_pstate_driver_lock); | |
1021 | ret = amd_pstate_show_status(buf); | |
1022 | mutex_unlock(&amd_pstate_driver_lock); | |
1023 | ||
1024 | return ret; | |
1025 | } | |
1026 | ||
5e720f8c | 1027 | static ssize_t status_store(struct device *a, struct device_attribute *b, |
abd61c08 PY |
1028 | const char *buf, size_t count) |
1029 | { | |
1030 | char *p = memchr(buf, '\n', count); | |
1031 | int ret; | |
1032 | ||
1033 | mutex_lock(&amd_pstate_driver_lock); | |
1034 | ret = amd_pstate_update_status(buf, p ? p - buf : count); | |
1035 | mutex_unlock(&amd_pstate_driver_lock); | |
1036 | ||
1037 | return ret < 0 ? ret : count; | |
1038 | } | |
1039 | ||
ec4e3326 HR |
1040 | cpufreq_freq_attr_ro(amd_pstate_max_freq); |
1041 | cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); | |
1042 | ||
3ad7fde1 | 1043 | cpufreq_freq_attr_ro(amd_pstate_highest_perf); |
ffa5096a PY |
1044 | cpufreq_freq_attr_rw(energy_performance_preference); |
1045 | cpufreq_freq_attr_ro(energy_performance_available_preferences); | |
5e720f8c | 1046 | static DEVICE_ATTR_RW(status); |
3ad7fde1 | 1047 | |
ec4e3326 HR |
1048 | static struct freq_attr *amd_pstate_attr[] = { |
1049 | &amd_pstate_max_freq, | |
1050 | &amd_pstate_lowest_nonlinear_freq, | |
3ad7fde1 | 1051 | &amd_pstate_highest_perf, |
ec4e3326 HR |
1052 | NULL, |
1053 | }; | |
1054 | ||
ffa5096a PY |
1055 | static struct freq_attr *amd_pstate_epp_attr[] = { |
1056 | &amd_pstate_max_freq, | |
1057 | &amd_pstate_lowest_nonlinear_freq, | |
1058 | &amd_pstate_highest_perf, | |
1059 | &energy_performance_preference, | |
1060 | &energy_performance_available_preferences, | |
1061 | NULL, | |
1062 | }; | |
1063 | ||
abd61c08 | 1064 | static struct attribute *pstate_global_attributes[] = { |
5e720f8c | 1065 | &dev_attr_status.attr, |
abd61c08 PY |
1066 | NULL |
1067 | }; | |
1068 | ||
1069 | static const struct attribute_group amd_pstate_global_attr_group = { | |
3666062b | 1070 | .name = "amd_pstate", |
abd61c08 PY |
1071 | .attrs = pstate_global_attributes, |
1072 | }; | |
1073 | ||
32f80b9a ML |
1074 | static bool amd_pstate_acpi_pm_profile_server(void) |
1075 | { | |
1076 | switch (acpi_gbl_FADT.preferred_profile) { | |
1077 | case PM_ENTERPRISE_SERVER: | |
1078 | case PM_SOHO_SERVER: | |
1079 | case PM_PERFORMANCE_SERVER: | |
1080 | return true; | |
1081 | } | |
1082 | return false; | |
1083 | } | |
1084 | ||
1085 | static bool amd_pstate_acpi_pm_profile_undefined(void) | |
1086 | { | |
1087 | if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) | |
1088 | return true; | |
1089 | if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) | |
1090 | return true; | |
1091 | return false; | |
1092 | } | |
1093 | ||
ffa5096a PY |
1094 | static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) |
1095 | { | |
1096 | int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; | |
1097 | struct amd_cpudata *cpudata; | |
1098 | struct device *dev; | |
ffa5096a PY |
1099 | u64 value; |
1100 | ||
1101 | /* | |
1102 | * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, | |
1103 | * which is ideal for initialization process. | |
1104 | */ | |
1105 | amd_perf_ctl_reset(policy->cpu); | |
1106 | dev = get_cpu_device(policy->cpu); | |
1107 | if (!dev) | |
7cca9a98 | 1108 | return -ENODEV; |
ffa5096a PY |
1109 | |
1110 | cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); | |
1111 | if (!cpudata) | |
1112 | return -ENOMEM; | |
1113 | ||
1114 | cpudata->cpu = policy->cpu; | |
1115 | cpudata->epp_policy = 0; | |
1116 | ||
7cca9a98 AB |
1117 | ret = amd_pstate_init_perf(cpudata); |
1118 | if (ret) | |
ffa5096a PY |
1119 | goto free_cpudata1; |
1120 | ||
1121 | min_freq = amd_get_min_freq(cpudata); | |
1122 | max_freq = amd_get_max_freq(cpudata); | |
1123 | nominal_freq = amd_get_nominal_freq(cpudata); | |
1124 | lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); | |
1125 | if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { | |
1126 | dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", | |
1127 | min_freq, max_freq); | |
1128 | ret = -EINVAL; | |
1129 | goto free_cpudata1; | |
1130 | } | |
1131 | ||
1132 | policy->cpuinfo.min_freq = min_freq; | |
1133 | policy->cpuinfo.max_freq = max_freq; | |
1134 | /* It will be updated by governor */ | |
1135 | policy->cur = policy->cpuinfo.min_freq; | |
1136 | ||
1137 | /* Initial processor data capability frequencies */ | |
1138 | cpudata->max_freq = max_freq; | |
1139 | cpudata->min_freq = min_freq; | |
1140 | cpudata->nominal_freq = nominal_freq; | |
1141 | cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; | |
1142 | ||
1143 | policy->driver_data = cpudata; | |
1144 | ||
1145 | cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); | |
1146 | ||
1147 | policy->min = policy->cpuinfo.min_freq; | |
1148 | policy->max = policy->cpuinfo.max_freq; | |
1149 | ||
1150 | /* | |
32f80b9a | 1151 | * Set the policy to provide a valid fallback value in case |
ffa5096a PY |
1152 | * the default cpufreq governor is neither powersave nor performance. |
1153 | */ | |
32f80b9a ML |
1154 | if (amd_pstate_acpi_pm_profile_server() || |
1155 | amd_pstate_acpi_pm_profile_undefined()) | |
1156 | policy->policy = CPUFREQ_POLICY_PERFORMANCE; | |
1157 | else | |
1158 | policy->policy = CPUFREQ_POLICY_POWERSAVE; | |
ffa5096a PY |
1159 | |
1160 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
ffa5096a PY |
1161 | ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); |
1162 | if (ret) | |
1163 | return ret; | |
1164 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
1165 | ||
1166 | ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); | |
1167 | if (ret) | |
1168 | return ret; | |
1169 | WRITE_ONCE(cpudata->cppc_cap1_cached, value); | |
1170 | } | |
1171 | amd_pstate_boost_init(cpudata); | |
1172 | ||
1173 | return 0; | |
1174 | ||
1175 | free_cpudata1: | |
1176 | kfree(cpudata); | |
1177 | return ret; | |
1178 | } | |
1179 | ||
1180 | static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) | |
1181 | { | |
1182 | pr_debug("CPU %d exiting\n", policy->cpu); | |
ffa5096a PY |
1183 | return 0; |
1184 | } | |
1185 | ||
1186 | static void amd_pstate_epp_init(unsigned int cpu) | |
1187 | { | |
1188 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); | |
1189 | struct amd_cpudata *cpudata = policy->driver_data; | |
1190 | u32 max_perf, min_perf; | |
1191 | u64 value; | |
1192 | s16 epp; | |
1193 | ||
1194 | max_perf = READ_ONCE(cpudata->highest_perf); | |
1195 | min_perf = READ_ONCE(cpudata->lowest_perf); | |
1196 | ||
1197 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1198 | ||
1199 | if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) | |
1200 | min_perf = max_perf; | |
1201 | ||
1202 | /* Initial min/max values for CPPC Performance Controls Register */ | |
1203 | value &= ~AMD_CPPC_MIN_PERF(~0L); | |
1204 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
1205 | ||
1206 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
1207 | value |= AMD_CPPC_MAX_PERF(max_perf); | |
1208 | ||
1209 | /* CPPC EPP feature require to set zero to the desire perf bit */ | |
1210 | value &= ~AMD_CPPC_DES_PERF(~0L); | |
1211 | value |= AMD_CPPC_DES_PERF(0); | |
1212 | ||
1213 | if (cpudata->epp_policy == cpudata->policy) | |
1214 | goto skip_epp; | |
1215 | ||
1216 | cpudata->epp_policy = cpudata->policy; | |
1217 | ||
6e9d1212 WK |
1218 | /* Get BIOS pre-defined epp value */ |
1219 | epp = amd_pstate_get_epp(cpudata, value); | |
1220 | if (epp < 0) { | |
1221 | /** | |
1222 | * This return value can only be negative for shared_memory | |
1223 | * systems where EPP register read/write not supported. | |
1224 | */ | |
1225 | goto skip_epp; | |
ffa5096a | 1226 | } |
6e9d1212 WK |
1227 | |
1228 | if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) | |
1229 | epp = 0; | |
1230 | ||
ffa5096a PY |
1231 | /* Set initial EPP value */ |
1232 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1233 | value &= ~GENMASK_ULL(31, 24); | |
1234 | value |= (u64)epp << 24; | |
1235 | } | |
1236 | ||
6e9d1212 | 1237 | WRITE_ONCE(cpudata->cppc_req_cached, value); |
7cca9a98 | 1238 | amd_pstate_set_epp(cpudata, epp); |
ffa5096a | 1239 | skip_epp: |
ffa5096a PY |
1240 | cpufreq_cpu_put(policy); |
1241 | } | |
1242 | ||
1243 | static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) | |
1244 | { | |
1245 | struct amd_cpudata *cpudata = policy->driver_data; | |
1246 | ||
1247 | if (!policy->cpuinfo.max_freq) | |
1248 | return -ENODEV; | |
1249 | ||
1250 | pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", | |
1251 | policy->cpuinfo.max_freq, policy->max); | |
1252 | ||
1253 | cpudata->policy = policy->policy; | |
1254 | ||
1255 | amd_pstate_epp_init(policy->cpu); | |
1256 | ||
1257 | return 0; | |
1258 | } | |
1259 | ||
d4da12f8 PY |
1260 | static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) |
1261 | { | |
1262 | struct cppc_perf_ctrls perf_ctrls; | |
1263 | u64 value, max_perf; | |
1264 | int ret; | |
1265 | ||
1266 | ret = amd_pstate_enable(true); | |
1267 | if (ret) | |
1268 | pr_err("failed to enable amd pstate during resume, return %d\n", ret); | |
1269 | ||
1270 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1271 | max_perf = READ_ONCE(cpudata->highest_perf); | |
1272 | ||
1273 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1274 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
1275 | } else { | |
1276 | perf_ctrls.max_perf = max_perf; | |
1277 | perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); | |
1278 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
1279 | } | |
1280 | } | |
1281 | ||
1282 | static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) | |
1283 | { | |
1284 | struct amd_cpudata *cpudata = policy->driver_data; | |
1285 | ||
1286 | pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); | |
1287 | ||
1288 | if (cppc_state == AMD_PSTATE_ACTIVE) { | |
1289 | amd_pstate_epp_reenable(cpudata); | |
1290 | cpudata->suspended = false; | |
1291 | } | |
1292 | ||
1293 | return 0; | |
1294 | } | |
1295 | ||
1296 | static void amd_pstate_epp_offline(struct cpufreq_policy *policy) | |
1297 | { | |
1298 | struct amd_cpudata *cpudata = policy->driver_data; | |
1299 | struct cppc_perf_ctrls perf_ctrls; | |
1300 | int min_perf; | |
1301 | u64 value; | |
1302 | ||
1303 | min_perf = READ_ONCE(cpudata->lowest_perf); | |
1304 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1305 | ||
1306 | mutex_lock(&amd_pstate_limits_lock); | |
1307 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1308 | cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; | |
1309 | ||
1310 | /* Set max perf same as min perf */ | |
1311 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
1312 | value |= AMD_CPPC_MAX_PERF(min_perf); | |
1313 | value &= ~AMD_CPPC_MIN_PERF(~0L); | |
1314 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
1315 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
1316 | } else { | |
1317 | perf_ctrls.desired_perf = 0; | |
1318 | perf_ctrls.max_perf = min_perf; | |
1319 | perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); | |
1320 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
1321 | } | |
1322 | mutex_unlock(&amd_pstate_limits_lock); | |
1323 | } | |
1324 | ||
1325 | static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) | |
1326 | { | |
1327 | struct amd_cpudata *cpudata = policy->driver_data; | |
1328 | ||
1329 | pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); | |
1330 | ||
1331 | if (cpudata->suspended) | |
1332 | return 0; | |
1333 | ||
1334 | if (cppc_state == AMD_PSTATE_ACTIVE) | |
1335 | amd_pstate_epp_offline(policy); | |
1336 | ||
1337 | return 0; | |
1338 | } | |
1339 | ||
ffa5096a PY |
1340 | static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) |
1341 | { | |
1342 | cpufreq_verify_within_cpu_limits(policy); | |
1343 | pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); | |
1344 | return 0; | |
1345 | } | |
1346 | ||
50ddd2f7 PY |
1347 | static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) |
1348 | { | |
1349 | struct amd_cpudata *cpudata = policy->driver_data; | |
1350 | int ret; | |
1351 | ||
1352 | /* avoid suspending when EPP is not enabled */ | |
1353 | if (cppc_state != AMD_PSTATE_ACTIVE) | |
1354 | return 0; | |
1355 | ||
1356 | /* set this flag to avoid setting core offline*/ | |
1357 | cpudata->suspended = true; | |
1358 | ||
1359 | /* disable CPPC in lowlevel firmware */ | |
1360 | ret = amd_pstate_enable(false); | |
1361 | if (ret) | |
1362 | pr_err("failed to suspend, return %d\n", ret); | |
1363 | ||
1364 | return 0; | |
1365 | } | |
1366 | ||
1367 | static int amd_pstate_epp_resume(struct cpufreq_policy *policy) | |
1368 | { | |
1369 | struct amd_cpudata *cpudata = policy->driver_data; | |
1370 | ||
1371 | if (cpudata->suspended) { | |
1372 | mutex_lock(&amd_pstate_limits_lock); | |
1373 | ||
1374 | /* enable amd pstate from suspend state*/ | |
1375 | amd_pstate_epp_reenable(cpudata); | |
1376 | ||
1377 | mutex_unlock(&amd_pstate_limits_lock); | |
1378 | ||
1379 | cpudata->suspended = false; | |
1380 | } | |
1381 | ||
1382 | return 0; | |
1383 | } | |
1384 | ||
ec437d71 HR |
1385 | static struct cpufreq_driver amd_pstate_driver = { |
1386 | .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, | |
1387 | .verify = amd_pstate_verify, | |
1388 | .target = amd_pstate_target, | |
4badf2eb | 1389 | .fast_switch = amd_pstate_fast_switch, |
ec437d71 HR |
1390 | .init = amd_pstate_cpu_init, |
1391 | .exit = amd_pstate_cpu_exit, | |
b376471f JS |
1392 | .suspend = amd_pstate_cpu_suspend, |
1393 | .resume = amd_pstate_cpu_resume, | |
41271016 | 1394 | .set_boost = amd_pstate_set_boost, |
ec437d71 | 1395 | .name = "amd-pstate", |
d8bee41d | 1396 | .attr = amd_pstate_attr, |
ec437d71 HR |
1397 | }; |
1398 | ||
ffa5096a PY |
1399 | static struct cpufreq_driver amd_pstate_epp_driver = { |
1400 | .flags = CPUFREQ_CONST_LOOPS, | |
1401 | .verify = amd_pstate_epp_verify_policy, | |
1402 | .setpolicy = amd_pstate_epp_set_policy, | |
1403 | .init = amd_pstate_epp_cpu_init, | |
1404 | .exit = amd_pstate_epp_cpu_exit, | |
d4da12f8 PY |
1405 | .offline = amd_pstate_epp_cpu_offline, |
1406 | .online = amd_pstate_epp_cpu_online, | |
50ddd2f7 PY |
1407 | .suspend = amd_pstate_epp_suspend, |
1408 | .resume = amd_pstate_epp_resume, | |
f4aad639 | 1409 | .name = "amd-pstate-epp", |
ffa5096a PY |
1410 | .attr = amd_pstate_epp_attr, |
1411 | }; | |
1412 | ||
c88ad30e ML |
1413 | static int __init amd_pstate_set_driver(int mode_idx) |
1414 | { | |
1415 | if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { | |
1416 | cppc_state = mode_idx; | |
1417 | if (cppc_state == AMD_PSTATE_DISABLE) | |
1418 | pr_info("driver is explicitly disabled\n"); | |
1419 | ||
1420 | if (cppc_state == AMD_PSTATE_ACTIVE) | |
1421 | current_pstate_driver = &amd_pstate_epp_driver; | |
1422 | ||
1423 | if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) | |
1424 | current_pstate_driver = &amd_pstate_driver; | |
1425 | ||
1426 | return 0; | |
1427 | } | |
1428 | ||
1429 | return -EINVAL; | |
1430 | } | |
1431 | ||
ec437d71 HR |
1432 | static int __init amd_pstate_init(void) |
1433 | { | |
3666062b | 1434 | struct device *dev_root; |
ec437d71 HR |
1435 | int ret; |
1436 | ||
1437 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) | |
1438 | return -ENODEV; | |
1439 | ||
1440 | if (!acpi_cpc_valid()) { | |
a2a9d185 | 1441 | pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); |
ec437d71 HR |
1442 | return -ENODEV; |
1443 | } | |
1444 | ||
1445 | /* don't keep reloading if cpufreq_driver exists */ | |
1446 | if (cpufreq_get_current_driver()) | |
1447 | return -EEXIST; | |
1448 | ||
c88ad30e ML |
1449 | switch (cppc_state) { |
1450 | case AMD_PSTATE_UNDEFINED: | |
1451 | /* Disable on the following configs by default: | |
1452 | * 1. Undefined platforms | |
1453 | * 2. Server platforms | |
1454 | * 3. Shared memory designs | |
1455 | */ | |
1456 | if (amd_pstate_acpi_pm_profile_undefined() || | |
1457 | amd_pstate_acpi_pm_profile_server() || | |
1458 | !boot_cpu_has(X86_FEATURE_CPPC)) { | |
1459 | pr_info("driver load is disabled, boot with specific mode to enable this\n"); | |
1460 | return -ENODEV; | |
1461 | } | |
1462 | ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE); | |
1463 | if (ret) | |
1464 | return ret; | |
1465 | break; | |
1466 | case AMD_PSTATE_DISABLE: | |
1467 | return -ENODEV; | |
1468 | case AMD_PSTATE_PASSIVE: | |
1469 | case AMD_PSTATE_ACTIVE: | |
1470 | case AMD_PSTATE_GUIDED: | |
1471 | break; | |
1472 | default: | |
1473 | return -EINVAL; | |
1474 | } | |
1475 | ||
ec437d71 | 1476 | /* capability check */ |
e059c184 HR |
1477 | if (boot_cpu_has(X86_FEATURE_CPPC)) { |
1478 | pr_debug("AMD CPPC MSR based functionality is supported\n"); | |
2dd6d0eb | 1479 | if (cppc_state != AMD_PSTATE_ACTIVE) |
ffa5096a | 1480 | current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; |
202e683d PY |
1481 | } else { |
1482 | pr_debug("AMD CPPC shared memory based functionality is supported\n"); | |
e059c184 HR |
1483 | static_call_update(amd_pstate_enable, cppc_enable); |
1484 | static_call_update(amd_pstate_init_perf, cppc_init_perf); | |
1485 | static_call_update(amd_pstate_update_perf, cppc_update_perf); | |
ec437d71 HR |
1486 | } |
1487 | ||
1488 | /* enable amd pstate feature */ | |
1489 | ret = amd_pstate_enable(true); | |
1490 | if (ret) { | |
ffa5096a | 1491 | pr_err("failed to enable with return %d\n", ret); |
ec437d71 HR |
1492 | return ret; |
1493 | } | |
1494 | ||
ffa5096a | 1495 | ret = cpufreq_register_driver(current_pstate_driver); |
ec437d71 | 1496 | if (ret) |
ffa5096a | 1497 | pr_err("failed to register with return %d\n", ret); |
ec437d71 | 1498 | |
3666062b GKH |
1499 | dev_root = bus_get_dev_root(&cpu_subsys); |
1500 | if (dev_root) { | |
1501 | ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); | |
1502 | put_device(dev_root); | |
1503 | if (ret) { | |
1504 | pr_err("sysfs attribute export failed with error %d.\n", ret); | |
1505 | goto global_attr_free; | |
1506 | } | |
abd61c08 PY |
1507 | } |
1508 | ||
1509 | return ret; | |
1510 | ||
1511 | global_attr_free: | |
abd61c08 | 1512 | cpufreq_unregister_driver(current_pstate_driver); |
ec437d71 HR |
1513 | return ret; |
1514 | } | |
456ca88d | 1515 | device_initcall(amd_pstate_init); |
ec437d71 | 1516 | |
202e683d PY |
1517 | static int __init amd_pstate_param(char *str) |
1518 | { | |
36c5014e WK |
1519 | size_t size; |
1520 | int mode_idx; | |
1521 | ||
202e683d PY |
1522 | if (!str) |
1523 | return -EINVAL; | |
1524 | ||
36c5014e WK |
1525 | size = strlen(str); |
1526 | mode_idx = get_mode_idx_from_str(str, size); | |
202e683d | 1527 | |
c88ad30e | 1528 | return amd_pstate_set_driver(mode_idx); |
202e683d PY |
1529 | } |
1530 | early_param("amd_pstate", amd_pstate_param); | |
1531 | ||
ec437d71 HR |
1532 | MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); |
1533 | MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); |