Commit | Line | Data |
---|---|---|
ec437d71 HR |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* | |
3 | * amd-pstate.c - AMD Processor P-state Frequency Driver | |
4 | * | |
5 | * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. | |
6 | * | |
7 | * Author: Huang Rui <ray.huang@amd.com> | |
8 | * | |
9 | * AMD P-State introduces a new CPU performance scaling design for AMD | |
10 | * processors using the ACPI Collaborative Performance and Power Control (CPPC) | |
11 | * feature which works with the AMD SMU firmware providing a finer grained | |
12 | * frequency control range. It is to replace the legacy ACPI P-States control, | |
13 | * allows a flexible, low-latency interface for the Linux kernel to directly | |
14 | * communicate the performance hints to hardware. | |
15 | * | |
16 | * AMD P-State is supported on recent AMD Zen base CPU series include some of | |
17 | * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD | |
18 | * P-State supported system. And there are two types of hardware implementations | |
19 | * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. | |
20 | * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. | |
21 | */ | |
22 | ||
23 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
24 | ||
25 | #include <linux/kernel.h> | |
26 | #include <linux/module.h> | |
27 | #include <linux/init.h> | |
28 | #include <linux/smp.h> | |
29 | #include <linux/sched.h> | |
30 | #include <linux/cpufreq.h> | |
31 | #include <linux/compiler.h> | |
32 | #include <linux/dmi.h> | |
33 | #include <linux/slab.h> | |
34 | #include <linux/acpi.h> | |
35 | #include <linux/io.h> | |
36 | #include <linux/delay.h> | |
37 | #include <linux/uaccess.h> | |
38 | #include <linux/static_call.h> | |
f1375ec1 | 39 | #include <linux/amd-pstate.h> |
f3a05239 | 40 | #include <linux/topology.h> |
ec437d71 HR |
41 | |
42 | #include <acpi/processor.h> | |
43 | #include <acpi/cppc_acpi.h> | |
44 | ||
45 | #include <asm/msr.h> | |
46 | #include <asm/processor.h> | |
47 | #include <asm/cpufeature.h> | |
48 | #include <asm/cpu_device_id.h> | |
60e10f89 | 49 | #include "amd-pstate-trace.h" |
ec437d71 | 50 | |
ca08e46d PY |
51 | #define AMD_PSTATE_TRANSITION_LATENCY 20000 |
52 | #define AMD_PSTATE_TRANSITION_DELAY 1000 | |
bf202e65 PY |
53 | #define CPPC_HIGHEST_PERF_PERFORMANCE 196 |
54 | #define CPPC_HIGHEST_PERF_DEFAULT 166 | |
ec437d71 | 55 | |
e059c184 HR |
56 | /* |
57 | * TODO: We need more time to fine tune processors with shared memory solution | |
58 | * with community together. | |
59 | * | |
60 | * There are some performance drops on the CPU benchmarks which reports from | |
61 | * Suse. We are co-working with them to fine tune the shared memory solution. So | |
62 | * we disable it by default to go acpi-cpufreq on these processors and add a | |
63 | * module parameter to be able to enable it manually for debugging. | |
64 | */ | |
ffa5096a | 65 | static struct cpufreq_driver *current_pstate_driver; |
ec437d71 | 66 | static struct cpufreq_driver amd_pstate_driver; |
ffa5096a | 67 | static struct cpufreq_driver amd_pstate_epp_driver; |
c88ad30e | 68 | static int cppc_state = AMD_PSTATE_UNDEFINED; |
217e6778 | 69 | static bool cppc_enabled; |
f3a05239 | 70 | static bool amd_pstate_prefcore = true; |
eb8b6c36 | 71 | static struct quirk_entry *quirks; |
36c5014e | 72 | |
ffa5096a PY |
73 | /* |
74 | * AMD Energy Preference Performance (EPP) | |
75 | * The EPP is used in the CCLK DPM controller to drive | |
76 | * the frequency that a core is going to operate during | |
77 | * short periods of activity. EPP values will be utilized for | |
78 | * different OS profiles (balanced, performance, power savings) | |
79 | * display strings corresponding to EPP index in the | |
80 | * energy_perf_strings[] | |
81 | * index String | |
82 | *------------------------------------- | |
83 | * 0 default | |
84 | * 1 performance | |
85 | * 2 balance_performance | |
86 | * 3 balance_power | |
87 | * 4 power | |
88 | */ | |
89 | enum energy_perf_value_index { | |
90 | EPP_INDEX_DEFAULT = 0, | |
91 | EPP_INDEX_PERFORMANCE, | |
92 | EPP_INDEX_BALANCE_PERFORMANCE, | |
93 | EPP_INDEX_BALANCE_POWERSAVE, | |
94 | EPP_INDEX_POWERSAVE, | |
95 | }; | |
96 | ||
97 | static const char * const energy_perf_strings[] = { | |
98 | [EPP_INDEX_DEFAULT] = "default", | |
99 | [EPP_INDEX_PERFORMANCE] = "performance", | |
100 | [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", | |
101 | [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", | |
102 | [EPP_INDEX_POWERSAVE] = "power", | |
103 | NULL | |
104 | }; | |
105 | ||
106 | static unsigned int epp_values[] = { | |
107 | [EPP_INDEX_DEFAULT] = 0, | |
108 | [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, | |
109 | [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, | |
110 | [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, | |
111 | [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, | |
112 | }; | |
113 | ||
3ca7bc81 WK |
114 | typedef int (*cppc_mode_transition_fn)(int); |
115 | ||
eb8b6c36 PY |
116 | static struct quirk_entry quirk_amd_7k62 = { |
117 | .nominal_freq = 2600, | |
118 | .lowest_freq = 550, | |
119 | }; | |
120 | ||
121 | static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi) | |
122 | { | |
123 | /** | |
124 | * match the broken bios for family 17h processor support CPPC V2 | |
125 | * broken BIOS lack of nominal_freq and lowest_freq capabilities | |
126 | * definition in ACPI tables | |
127 | */ | |
128 | if (boot_cpu_has(X86_FEATURE_ZEN2)) { | |
129 | quirks = dmi->driver_data; | |
130 | pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident); | |
131 | return 1; | |
132 | } | |
133 | ||
134 | return 0; | |
135 | } | |
136 | ||
137 | static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = { | |
138 | { | |
139 | .callback = dmi_matched_7k62_bios_bug, | |
140 | .ident = "AMD EPYC 7K62", | |
141 | .matches = { | |
142 | DMI_MATCH(DMI_BIOS_VERSION, "5.14"), | |
143 | DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"), | |
144 | }, | |
145 | .driver_data = &quirk_amd_7k62, | |
146 | }, | |
147 | {} | |
148 | }; | |
149 | MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table); | |
150 | ||
36c5014e WK |
151 | static inline int get_mode_idx_from_str(const char *str, size_t size) |
152 | { | |
153 | int i; | |
154 | ||
155 | for (i=0; i < AMD_PSTATE_MAX; i++) { | |
156 | if (!strncmp(str, amd_pstate_mode_string[i], size)) | |
157 | return i; | |
158 | } | |
159 | return -EINVAL; | |
160 | } | |
ec437d71 | 161 | |
ffa5096a PY |
162 | static DEFINE_MUTEX(amd_pstate_limits_lock); |
163 | static DEFINE_MUTEX(amd_pstate_driver_lock); | |
164 | ||
165 | static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) | |
166 | { | |
167 | u64 epp; | |
168 | int ret; | |
169 | ||
170 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
171 | if (!cppc_req_cached) { | |
172 | epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, | |
173 | &cppc_req_cached); | |
174 | if (epp) | |
175 | return epp; | |
176 | } | |
177 | epp = (cppc_req_cached >> 24) & 0xFF; | |
178 | } else { | |
179 | ret = cppc_get_epp_perf(cpudata->cpu, &epp); | |
180 | if (ret < 0) { | |
181 | pr_debug("Could not retrieve energy perf value (%d)\n", ret); | |
182 | return -EIO; | |
183 | } | |
184 | } | |
185 | ||
186 | return (s16)(epp & 0xff); | |
187 | } | |
188 | ||
189 | static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) | |
190 | { | |
191 | s16 epp; | |
192 | int index = -EINVAL; | |
193 | ||
194 | epp = amd_pstate_get_epp(cpudata, 0); | |
195 | if (epp < 0) | |
196 | return epp; | |
197 | ||
198 | switch (epp) { | |
199 | case AMD_CPPC_EPP_PERFORMANCE: | |
200 | index = EPP_INDEX_PERFORMANCE; | |
201 | break; | |
202 | case AMD_CPPC_EPP_BALANCE_PERFORMANCE: | |
203 | index = EPP_INDEX_BALANCE_PERFORMANCE; | |
204 | break; | |
205 | case AMD_CPPC_EPP_BALANCE_POWERSAVE: | |
206 | index = EPP_INDEX_BALANCE_POWERSAVE; | |
207 | break; | |
208 | case AMD_CPPC_EPP_POWERSAVE: | |
209 | index = EPP_INDEX_POWERSAVE; | |
210 | break; | |
211 | default: | |
212 | break; | |
213 | } | |
214 | ||
215 | return index; | |
216 | } | |
217 | ||
218 | static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) | |
219 | { | |
220 | int ret; | |
221 | struct cppc_perf_ctrls perf_ctrls; | |
222 | ||
223 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
224 | u64 value = READ_ONCE(cpudata->cppc_req_cached); | |
225 | ||
226 | value &= ~GENMASK_ULL(31, 24); | |
227 | value |= (u64)epp << 24; | |
228 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
229 | ||
230 | ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
231 | if (!ret) | |
232 | cpudata->epp_cached = epp; | |
233 | } else { | |
234 | perf_ctrls.energy_perf = epp; | |
235 | ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); | |
236 | if (ret) { | |
237 | pr_debug("failed to set energy perf value (%d)\n", ret); | |
238 | return ret; | |
239 | } | |
240 | cpudata->epp_cached = epp; | |
241 | } | |
242 | ||
243 | return ret; | |
244 | } | |
245 | ||
246 | static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, | |
247 | int pref_index) | |
248 | { | |
249 | int epp = -EINVAL; | |
250 | int ret; | |
251 | ||
252 | if (!pref_index) { | |
253 | pr_debug("EPP pref_index is invalid\n"); | |
254 | return -EINVAL; | |
255 | } | |
256 | ||
257 | if (epp == -EINVAL) | |
258 | epp = epp_values[pref_index]; | |
259 | ||
260 | if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { | |
261 | pr_debug("EPP cannot be set under performance policy\n"); | |
262 | return -EBUSY; | |
263 | } | |
264 | ||
265 | ret = amd_pstate_set_epp(cpudata, epp); | |
266 | ||
267 | return ret; | |
268 | } | |
269 | ||
e059c184 | 270 | static inline int pstate_enable(bool enable) |
ec437d71 | 271 | { |
217e6778 WK |
272 | int ret, cpu; |
273 | unsigned long logical_proc_id_mask = 0; | |
274 | ||
275 | if (enable == cppc_enabled) | |
276 | return 0; | |
277 | ||
278 | for_each_present_cpu(cpu) { | |
279 | unsigned long logical_id = topology_logical_die_id(cpu); | |
280 | ||
281 | if (test_bit(logical_id, &logical_proc_id_mask)) | |
282 | continue; | |
283 | ||
284 | set_bit(logical_id, &logical_proc_id_mask); | |
285 | ||
286 | ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE, | |
287 | enable); | |
288 | if (ret) | |
289 | return ret; | |
290 | } | |
291 | ||
292 | cppc_enabled = enable; | |
293 | return 0; | |
ec437d71 HR |
294 | } |
295 | ||
e059c184 HR |
296 | static int cppc_enable(bool enable) |
297 | { | |
298 | int cpu, ret = 0; | |
ffa5096a | 299 | struct cppc_perf_ctrls perf_ctrls; |
e059c184 | 300 | |
217e6778 WK |
301 | if (enable == cppc_enabled) |
302 | return 0; | |
303 | ||
e059c184 HR |
304 | for_each_present_cpu(cpu) { |
305 | ret = cppc_set_enable(cpu, enable); | |
306 | if (ret) | |
307 | return ret; | |
ffa5096a PY |
308 | |
309 | /* Enable autonomous mode for EPP */ | |
310 | if (cppc_state == AMD_PSTATE_ACTIVE) { | |
311 | /* Set desired perf as zero to allow EPP firmware control */ | |
312 | perf_ctrls.desired_perf = 0; | |
313 | ret = cppc_set_perf(cpu, &perf_ctrls); | |
314 | if (ret) | |
315 | return ret; | |
316 | } | |
e059c184 HR |
317 | } |
318 | ||
217e6778 | 319 | cppc_enabled = enable; |
e059c184 HR |
320 | return ret; |
321 | } | |
322 | ||
323 | DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); | |
324 | ||
325 | static inline int amd_pstate_enable(bool enable) | |
326 | { | |
327 | return static_call(amd_pstate_enable)(enable); | |
328 | } | |
329 | ||
bf202e65 PY |
330 | static u32 amd_pstate_highest_perf_set(struct amd_cpudata *cpudata) |
331 | { | |
332 | struct cpuinfo_x86 *c = &cpu_data(0); | |
333 | ||
334 | /* | |
335 | * For AMD CPUs with Family ID 19H and Model ID range 0x70 to 0x7f, | |
336 | * the highest performance level is set to 196. | |
337 | * https://bugzilla.kernel.org/show_bug.cgi?id=218759 | |
338 | */ | |
339 | if (c->x86 == 0x19 && (c->x86_model >= 0x70 && c->x86_model <= 0x7f)) | |
340 | return CPPC_HIGHEST_PERF_PERFORMANCE; | |
341 | ||
342 | return CPPC_HIGHEST_PERF_DEFAULT; | |
343 | } | |
344 | ||
e059c184 | 345 | static int pstate_init_perf(struct amd_cpudata *cpudata) |
ec437d71 HR |
346 | { |
347 | u64 cap1; | |
bedadcfb | 348 | u32 highest_perf; |
ec437d71 HR |
349 | |
350 | int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, | |
351 | &cap1); | |
352 | if (ret) | |
353 | return ret; | |
354 | ||
f3a05239 ML |
355 | /* For platforms that do not support the preferred core feature, the |
356 | * highest_pef may be configured with 166 or 255, to avoid max frequency | |
357 | * calculated wrongly. we take the AMD_CPPC_HIGHEST_PERF(cap1) value as | |
358 | * the default max perf. | |
ec437d71 | 359 | */ |
f3a05239 | 360 | if (cpudata->hw_prefcore) |
bf202e65 | 361 | highest_perf = amd_pstate_highest_perf_set(cpudata); |
f3a05239 | 362 | else |
bedadcfb PY |
363 | highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); |
364 | ||
365 | WRITE_ONCE(cpudata->highest_perf, highest_perf); | |
febab20c | 366 | WRITE_ONCE(cpudata->max_limit_perf, highest_perf); |
ec437d71 HR |
367 | WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); |
368 | WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); | |
369 | WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); | |
e571a5e2 | 370 | WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1)); |
febab20c | 371 | WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1)); |
ec437d71 HR |
372 | return 0; |
373 | } | |
374 | ||
e059c184 HR |
375 | static int cppc_init_perf(struct amd_cpudata *cpudata) |
376 | { | |
377 | struct cppc_perf_caps cppc_perf; | |
bedadcfb | 378 | u32 highest_perf; |
e059c184 HR |
379 | |
380 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
381 | if (ret) | |
382 | return ret; | |
383 | ||
f3a05239 | 384 | if (cpudata->hw_prefcore) |
bf202e65 | 385 | highest_perf = amd_pstate_highest_perf_set(cpudata); |
f3a05239 | 386 | else |
bedadcfb PY |
387 | highest_perf = cppc_perf.highest_perf; |
388 | ||
389 | WRITE_ONCE(cpudata->highest_perf, highest_perf); | |
febab20c | 390 | WRITE_ONCE(cpudata->max_limit_perf, highest_perf); |
e059c184 HR |
391 | WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); |
392 | WRITE_ONCE(cpudata->lowest_nonlinear_perf, | |
393 | cppc_perf.lowest_nonlinear_perf); | |
394 | WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); | |
e571a5e2 | 395 | WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); |
febab20c | 396 | WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf); |
e059c184 | 397 | |
2dd6d0eb WK |
398 | if (cppc_state == AMD_PSTATE_ACTIVE) |
399 | return 0; | |
400 | ||
401 | ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf); | |
402 | if (ret) { | |
403 | pr_warn("failed to get auto_sel, ret: %d\n", ret); | |
404 | return 0; | |
405 | } | |
406 | ||
407 | ret = cppc_set_auto_sel(cpudata->cpu, | |
408 | (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); | |
409 | ||
410 | if (ret) | |
411 | pr_warn("failed to set auto_sel, ret: %d\n", ret); | |
412 | ||
413 | return ret; | |
e059c184 HR |
414 | } |
415 | ||
416 | DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); | |
417 | ||
418 | static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) | |
419 | { | |
420 | return static_call(amd_pstate_init_perf)(cpudata); | |
421 | } | |
422 | ||
423 | static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, | |
424 | u32 des_perf, u32 max_perf, bool fast_switch) | |
ec437d71 HR |
425 | { |
426 | if (fast_switch) | |
427 | wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); | |
428 | else | |
429 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, | |
430 | READ_ONCE(cpudata->cppc_req_cached)); | |
431 | } | |
432 | ||
e059c184 HR |
433 | static void cppc_update_perf(struct amd_cpudata *cpudata, |
434 | u32 min_perf, u32 des_perf, | |
435 | u32 max_perf, bool fast_switch) | |
436 | { | |
437 | struct cppc_perf_ctrls perf_ctrls; | |
438 | ||
439 | perf_ctrls.max_perf = max_perf; | |
440 | perf_ctrls.min_perf = min_perf; | |
441 | perf_ctrls.desired_perf = des_perf; | |
442 | ||
443 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
444 | } | |
445 | ||
446 | DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); | |
447 | ||
448 | static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, | |
449 | u32 min_perf, u32 des_perf, | |
450 | u32 max_perf, bool fast_switch) | |
451 | { | |
452 | static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, | |
453 | max_perf, fast_switch); | |
454 | } | |
455 | ||
23c296fb JS |
456 | static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) |
457 | { | |
458 | u64 aperf, mperf, tsc; | |
459 | unsigned long flags; | |
460 | ||
461 | local_irq_save(flags); | |
462 | rdmsrl(MSR_IA32_APERF, aperf); | |
463 | rdmsrl(MSR_IA32_MPERF, mperf); | |
464 | tsc = rdtsc(); | |
465 | ||
466 | if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { | |
467 | local_irq_restore(flags); | |
468 | return false; | |
469 | } | |
470 | ||
471 | local_irq_restore(flags); | |
472 | ||
473 | cpudata->cur.aperf = aperf; | |
474 | cpudata->cur.mperf = mperf; | |
475 | cpudata->cur.tsc = tsc; | |
476 | cpudata->cur.aperf -= cpudata->prev.aperf; | |
477 | cpudata->cur.mperf -= cpudata->prev.mperf; | |
478 | cpudata->cur.tsc -= cpudata->prev.tsc; | |
479 | ||
480 | cpudata->prev.aperf = aperf; | |
481 | cpudata->prev.mperf = mperf; | |
482 | cpudata->prev.tsc = tsc; | |
483 | ||
484 | cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); | |
485 | ||
486 | return true; | |
487 | } | |
488 | ||
ec437d71 | 489 | static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, |
2dd6d0eb | 490 | u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags) |
ec437d71 HR |
491 | { |
492 | u64 prev = READ_ONCE(cpudata->cppc_req_cached); | |
493 | u64 value = prev; | |
494 | ||
febab20c WK |
495 | min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, |
496 | cpudata->max_limit_perf); | |
497 | max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, | |
498 | cpudata->max_limit_perf); | |
0e9a8638 | 499 | des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); |
2dd6d0eb WK |
500 | |
501 | if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) { | |
502 | min_perf = des_perf; | |
503 | des_perf = 0; | |
504 | } | |
505 | ||
ec437d71 HR |
506 | value &= ~AMD_CPPC_MIN_PERF(~0L); |
507 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
508 | ||
509 | value &= ~AMD_CPPC_DES_PERF(~0L); | |
510 | value |= AMD_CPPC_DES_PERF(des_perf); | |
511 | ||
512 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
513 | value |= AMD_CPPC_MAX_PERF(max_perf); | |
514 | ||
23c296fb JS |
515 | if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { |
516 | trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, | |
517 | cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, | |
518 | cpudata->cpu, (value != prev), fast_switch); | |
519 | } | |
60e10f89 | 520 | |
ec437d71 HR |
521 | if (value == prev) |
522 | return; | |
523 | ||
524 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
525 | ||
526 | amd_pstate_update_perf(cpudata, min_perf, des_perf, | |
527 | max_perf, fast_switch); | |
528 | } | |
529 | ||
530 | static int amd_pstate_verify(struct cpufreq_policy_data *policy) | |
531 | { | |
532 | cpufreq_verify_within_cpu_limits(policy); | |
533 | ||
534 | return 0; | |
535 | } | |
536 | ||
febab20c WK |
537 | static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) |
538 | { | |
8164f743 | 539 | u32 max_limit_perf, min_limit_perf, lowest_perf; |
febab20c WK |
540 | struct amd_cpudata *cpudata = policy->driver_data; |
541 | ||
542 | max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); | |
543 | min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); | |
544 | ||
8164f743 ML |
545 | lowest_perf = READ_ONCE(cpudata->lowest_perf); |
546 | if (min_limit_perf < lowest_perf) | |
547 | min_limit_perf = lowest_perf; | |
548 | ||
549 | if (max_limit_perf < min_limit_perf) | |
550 | max_limit_perf = min_limit_perf; | |
551 | ||
febab20c WK |
552 | WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); |
553 | WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); | |
554 | WRITE_ONCE(cpudata->max_limit_freq, policy->max); | |
555 | WRITE_ONCE(cpudata->min_limit_freq, policy->min); | |
556 | ||
557 | return 0; | |
558 | } | |
559 | ||
4badf2eb GS |
560 | static int amd_pstate_update_freq(struct cpufreq_policy *policy, |
561 | unsigned int target_freq, bool fast_switch) | |
ec437d71 HR |
562 | { |
563 | struct cpufreq_freqs freqs; | |
564 | struct amd_cpudata *cpudata = policy->driver_data; | |
565 | unsigned long max_perf, min_perf, des_perf, cap_perf; | |
566 | ||
567 | if (!cpudata->max_freq) | |
568 | return -ENODEV; | |
569 | ||
febab20c WK |
570 | if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) |
571 | amd_pstate_update_min_max_limit(policy); | |
572 | ||
ec437d71 | 573 | cap_perf = READ_ONCE(cpudata->highest_perf); |
b185c505 | 574 | min_perf = READ_ONCE(cpudata->lowest_perf); |
ec437d71 HR |
575 | max_perf = cap_perf; |
576 | ||
577 | freqs.old = policy->cur; | |
578 | freqs.new = target_freq; | |
579 | ||
580 | des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, | |
581 | cpudata->max_freq); | |
582 | ||
4badf2eb GS |
583 | WARN_ON(fast_switch && !policy->fast_switch_enabled); |
584 | /* | |
585 | * If fast_switch is desired, then there aren't any registered | |
586 | * transition notifiers. See comment for | |
587 | * cpufreq_enable_fast_switch(). | |
588 | */ | |
589 | if (!fast_switch) | |
590 | cpufreq_freq_transition_begin(policy, &freqs); | |
591 | ||
ec437d71 | 592 | amd_pstate_update(cpudata, min_perf, des_perf, |
4badf2eb GS |
593 | max_perf, fast_switch, policy->governor->flags); |
594 | ||
595 | if (!fast_switch) | |
596 | cpufreq_freq_transition_end(policy, &freqs, false); | |
ec437d71 HR |
597 | |
598 | return 0; | |
599 | } | |
600 | ||
4badf2eb GS |
601 | static int amd_pstate_target(struct cpufreq_policy *policy, |
602 | unsigned int target_freq, | |
603 | unsigned int relation) | |
604 | { | |
605 | return amd_pstate_update_freq(policy, target_freq, false); | |
606 | } | |
607 | ||
608 | static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy, | |
609 | unsigned int target_freq) | |
610 | { | |
bb87be26 GS |
611 | if (!amd_pstate_update_freq(policy, target_freq, true)) |
612 | return target_freq; | |
613 | return policy->cur; | |
4badf2eb GS |
614 | } |
615 | ||
1d215f03 HR |
616 | static void amd_pstate_adjust_perf(unsigned int cpu, |
617 | unsigned long _min_perf, | |
618 | unsigned long target_perf, | |
619 | unsigned long capacity) | |
620 | { | |
621 | unsigned long max_perf, min_perf, des_perf, | |
3bf8c630 | 622 | cap_perf, lowest_nonlinear_perf, max_freq; |
1d215f03 HR |
623 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); |
624 | struct amd_cpudata *cpudata = policy->driver_data; | |
3bf8c630 | 625 | unsigned int target_freq; |
1d215f03 | 626 | |
febab20c WK |
627 | if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq) |
628 | amd_pstate_update_min_max_limit(policy); | |
629 | ||
630 | ||
1d215f03 HR |
631 | cap_perf = READ_ONCE(cpudata->highest_perf); |
632 | lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); | |
3bf8c630 | 633 | max_freq = READ_ONCE(cpudata->max_freq); |
1d215f03 HR |
634 | |
635 | des_perf = cap_perf; | |
636 | if (target_perf < capacity) | |
637 | des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); | |
638 | ||
b26ffbf8 | 639 | min_perf = READ_ONCE(cpudata->lowest_perf); |
1d215f03 HR |
640 | if (_min_perf < capacity) |
641 | min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); | |
642 | ||
643 | if (min_perf < lowest_nonlinear_perf) | |
644 | min_perf = lowest_nonlinear_perf; | |
645 | ||
646 | max_perf = cap_perf; | |
647 | if (max_perf < min_perf) | |
648 | max_perf = min_perf; | |
649 | ||
3bf8c630 WK |
650 | des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); |
651 | target_freq = div_u64(des_perf * max_freq, max_perf); | |
652 | policy->cur = target_freq; | |
653 | ||
2dd6d0eb WK |
654 | amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true, |
655 | policy->governor->flags); | |
4f3085f8 | 656 | cpufreq_cpu_put(policy); |
1d215f03 HR |
657 | } |
658 | ||
41271016 HR |
659 | static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) |
660 | { | |
661 | struct amd_cpudata *cpudata = policy->driver_data; | |
662 | int ret; | |
663 | ||
664 | if (!cpudata->boost_supported) { | |
665 | pr_err("Boost mode is not supported by this processor or SBIOS\n"); | |
666 | return -EINVAL; | |
667 | } | |
668 | ||
669 | if (state) | |
670 | policy->cpuinfo.max_freq = cpudata->max_freq; | |
671 | else | |
672 | policy->cpuinfo.max_freq = cpudata->nominal_freq; | |
673 | ||
674 | policy->max = policy->cpuinfo.max_freq; | |
675 | ||
676 | ret = freq_qos_update_request(&cpudata->req[1], | |
677 | policy->cpuinfo.max_freq); | |
678 | if (ret < 0) | |
679 | return ret; | |
680 | ||
681 | return 0; | |
682 | } | |
683 | ||
684 | static void amd_pstate_boost_init(struct amd_cpudata *cpudata) | |
685 | { | |
686 | u32 highest_perf, nominal_perf; | |
687 | ||
688 | highest_perf = READ_ONCE(cpudata->highest_perf); | |
689 | nominal_perf = READ_ONCE(cpudata->nominal_perf); | |
690 | ||
691 | if (highest_perf <= nominal_perf) | |
692 | return; | |
693 | ||
694 | cpudata->boost_supported = true; | |
ffa5096a | 695 | current_pstate_driver->boost_enabled = true; |
41271016 HR |
696 | } |
697 | ||
919f4557 WK |
698 | static void amd_perf_ctl_reset(unsigned int cpu) |
699 | { | |
700 | wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); | |
701 | } | |
702 | ||
f3a05239 ML |
703 | /* |
704 | * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks | |
705 | * due to locking, so queue the work for later. | |
706 | */ | |
707 | static void amd_pstste_sched_prefcore_workfn(struct work_struct *work) | |
708 | { | |
709 | sched_set_itmt_support(); | |
710 | } | |
711 | static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn); | |
712 | ||
713 | /* | |
714 | * Get the highest performance register value. | |
715 | * @cpu: CPU from which to get highest performance. | |
716 | * @highest_perf: Return address. | |
717 | * | |
718 | * Return: 0 for success, -EIO otherwise. | |
719 | */ | |
720 | static int amd_pstate_get_highest_perf(int cpu, u32 *highest_perf) | |
721 | { | |
722 | int ret; | |
723 | ||
724 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
725 | u64 cap1; | |
726 | ||
727 | ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1); | |
728 | if (ret) | |
729 | return ret; | |
730 | WRITE_ONCE(*highest_perf, AMD_CPPC_HIGHEST_PERF(cap1)); | |
731 | } else { | |
732 | u64 cppc_highest_perf; | |
733 | ||
734 | ret = cppc_get_highest_perf(cpu, &cppc_highest_perf); | |
735 | if (ret) | |
736 | return ret; | |
737 | WRITE_ONCE(*highest_perf, cppc_highest_perf); | |
738 | } | |
739 | ||
740 | return (ret); | |
741 | } | |
742 | ||
743 | #define CPPC_MAX_PERF U8_MAX | |
744 | ||
745 | static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata) | |
746 | { | |
747 | int ret, prio; | |
748 | u32 highest_perf; | |
749 | ||
750 | ret = amd_pstate_get_highest_perf(cpudata->cpu, &highest_perf); | |
751 | if (ret) | |
752 | return; | |
753 | ||
754 | cpudata->hw_prefcore = true; | |
755 | /* check if CPPC preferred core feature is enabled*/ | |
756 | if (highest_perf < CPPC_MAX_PERF) | |
757 | prio = (int)highest_perf; | |
758 | else { | |
759 | pr_debug("AMD CPPC preferred core is unsupported!\n"); | |
760 | cpudata->hw_prefcore = false; | |
761 | return; | |
762 | } | |
763 | ||
764 | if (!amd_pstate_prefcore) | |
765 | return; | |
766 | ||
767 | /* | |
768 | * The priorities can be set regardless of whether or not | |
769 | * sched_set_itmt_support(true) has been called and it is valid to | |
770 | * update them at any time after it has been called. | |
771 | */ | |
772 | sched_set_itmt_core_prio(prio, cpudata->cpu); | |
773 | ||
774 | schedule_work(&sched_prefcore_work); | |
775 | } | |
776 | ||
e571a5e2 ML |
777 | static void amd_pstate_update_limits(unsigned int cpu) |
778 | { | |
779 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); | |
780 | struct amd_cpudata *cpudata = policy->driver_data; | |
781 | u32 prev_high = 0, cur_high = 0; | |
782 | int ret; | |
783 | bool highest_perf_changed = false; | |
784 | ||
785 | mutex_lock(&amd_pstate_driver_lock); | |
786 | if ((!amd_pstate_prefcore) || (!cpudata->hw_prefcore)) | |
787 | goto free_cpufreq_put; | |
788 | ||
789 | ret = amd_pstate_get_highest_perf(cpu, &cur_high); | |
790 | if (ret) | |
791 | goto free_cpufreq_put; | |
792 | ||
793 | prev_high = READ_ONCE(cpudata->prefcore_ranking); | |
794 | if (prev_high != cur_high) { | |
795 | highest_perf_changed = true; | |
796 | WRITE_ONCE(cpudata->prefcore_ranking, cur_high); | |
797 | ||
798 | if (cur_high < CPPC_MAX_PERF) | |
799 | sched_set_itmt_core_prio((int)cur_high, cpu); | |
800 | } | |
801 | ||
802 | free_cpufreq_put: | |
803 | cpufreq_cpu_put(policy); | |
804 | ||
805 | if (!highest_perf_changed) | |
806 | cpufreq_update_policy(cpu); | |
807 | ||
808 | mutex_unlock(&amd_pstate_driver_lock); | |
809 | } | |
810 | ||
5131a3ca | 811 | /* |
069a2bb8 PY |
812 | * Get pstate transition delay time from ACPI tables that firmware set |
813 | * instead of using hardcode value directly. | |
814 | */ | |
815 | static u32 amd_pstate_get_transition_delay_us(unsigned int cpu) | |
816 | { | |
817 | u32 transition_delay_ns; | |
818 | ||
819 | transition_delay_ns = cppc_get_transition_latency(cpu); | |
820 | if (transition_delay_ns == CPUFREQ_ETERNAL) | |
821 | return AMD_PSTATE_TRANSITION_DELAY; | |
822 | ||
823 | return transition_delay_ns / NSEC_PER_USEC; | |
824 | } | |
825 | ||
5131a3ca | 826 | /* |
069a2bb8 PY |
827 | * Get pstate transition latency value from ACPI tables that firmware |
828 | * set instead of using hardcode value directly. | |
829 | */ | |
830 | static u32 amd_pstate_get_transition_latency(unsigned int cpu) | |
831 | { | |
832 | u32 transition_latency; | |
833 | ||
834 | transition_latency = cppc_get_transition_latency(cpu); | |
835 | if (transition_latency == CPUFREQ_ETERNAL) | |
836 | return AMD_PSTATE_TRANSITION_LATENCY; | |
837 | ||
838 | return transition_latency; | |
839 | } | |
840 | ||
5131a3ca | 841 | /* |
5547c0eb PY |
842 | * amd_pstate_init_freq: Initialize the max_freq, min_freq, |
843 | * nominal_freq and lowest_nonlinear_freq for | |
844 | * the @cpudata object. | |
845 | * | |
846 | * Requires: highest_perf, lowest_perf, nominal_perf and | |
847 | * lowest_nonlinear_perf members of @cpudata to be | |
848 | * initialized. | |
849 | * | |
850 | * Returns 0 on success, non-zero value on failure. | |
851 | */ | |
852 | static int amd_pstate_init_freq(struct amd_cpudata *cpudata) | |
853 | { | |
854 | int ret; | |
855 | u32 min_freq; | |
856 | u32 highest_perf, max_freq; | |
857 | u32 nominal_perf, nominal_freq; | |
858 | u32 lowest_nonlinear_perf, lowest_nonlinear_freq; | |
859 | u32 boost_ratio, lowest_nonlinear_ratio; | |
860 | struct cppc_perf_caps cppc_perf; | |
861 | ||
5547c0eb PY |
862 | ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); |
863 | if (ret) | |
864 | return ret; | |
865 | ||
eb8b6c36 PY |
866 | if (quirks && quirks->lowest_freq) |
867 | min_freq = quirks->lowest_freq * 1000; | |
868 | else | |
869 | min_freq = cppc_perf.lowest_freq * 1000; | |
870 | ||
871 | if (quirks && quirks->nominal_freq) | |
872 | nominal_freq = quirks->nominal_freq ; | |
873 | else | |
874 | nominal_freq = cppc_perf.nominal_freq; | |
875 | ||
5547c0eb PY |
876 | nominal_perf = READ_ONCE(cpudata->nominal_perf); |
877 | ||
878 | highest_perf = READ_ONCE(cpudata->highest_perf); | |
879 | boost_ratio = div_u64(highest_perf << SCHED_CAPACITY_SHIFT, nominal_perf); | |
880 | max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000; | |
881 | ||
882 | lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); | |
883 | lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, | |
884 | nominal_perf); | |
885 | lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT) * 1000; | |
886 | ||
887 | WRITE_ONCE(cpudata->min_freq, min_freq); | |
888 | WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq); | |
889 | WRITE_ONCE(cpudata->nominal_freq, nominal_freq); | |
890 | WRITE_ONCE(cpudata->max_freq, max_freq); | |
891 | ||
892 | return 0; | |
893 | } | |
894 | ||
ec437d71 HR |
895 | static int amd_pstate_cpu_init(struct cpufreq_policy *policy) |
896 | { | |
5c3fd1ed | 897 | int min_freq, max_freq, nominal_freq, ret; |
ec437d71 HR |
898 | struct device *dev; |
899 | struct amd_cpudata *cpudata; | |
900 | ||
919f4557 WK |
901 | /* |
902 | * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, | |
903 | * which is ideal for initialization process. | |
904 | */ | |
905 | amd_perf_ctl_reset(policy->cpu); | |
ec437d71 HR |
906 | dev = get_cpu_device(policy->cpu); |
907 | if (!dev) | |
908 | return -ENODEV; | |
909 | ||
910 | cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); | |
911 | if (!cpudata) | |
912 | return -ENOMEM; | |
913 | ||
914 | cpudata->cpu = policy->cpu; | |
915 | ||
f3a05239 ML |
916 | amd_pstate_init_prefcore(cpudata); |
917 | ||
ec437d71 HR |
918 | ret = amd_pstate_init_perf(cpudata); |
919 | if (ret) | |
41271016 | 920 | goto free_cpudata1; |
ec437d71 | 921 | |
5547c0eb PY |
922 | ret = amd_pstate_init_freq(cpudata); |
923 | if (ret) | |
924 | goto free_cpudata1; | |
925 | ||
3cbbe887 GS |
926 | min_freq = READ_ONCE(cpudata->min_freq); |
927 | max_freq = READ_ONCE(cpudata->max_freq); | |
928 | nominal_freq = READ_ONCE(cpudata->nominal_freq); | |
ec437d71 | 929 | |
2ddb8a39 PY |
930 | if (min_freq <= 0 || max_freq <= 0 || |
931 | nominal_freq <= 0 || min_freq > max_freq) { | |
932 | dev_err(dev, | |
933 | "min_freq(%d) or max_freq(%d) or nominal_freq (%d) value is incorrect, check _CPC in ACPI tables\n", | |
934 | min_freq, max_freq, nominal_freq); | |
ec437d71 | 935 | ret = -EINVAL; |
41271016 | 936 | goto free_cpudata1; |
ec437d71 HR |
937 | } |
938 | ||
069a2bb8 PY |
939 | policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu); |
940 | policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu); | |
ec437d71 HR |
941 | |
942 | policy->min = min_freq; | |
943 | policy->max = max_freq; | |
944 | ||
945 | policy->cpuinfo.min_freq = min_freq; | |
946 | policy->cpuinfo.max_freq = max_freq; | |
947 | ||
948 | /* It will be updated by governor */ | |
949 | policy->cur = policy->cpuinfo.min_freq; | |
950 | ||
e059c184 HR |
951 | if (boot_cpu_has(X86_FEATURE_CPPC)) |
952 | policy->fast_switch_possible = true; | |
1d215f03 | 953 | |
41271016 HR |
954 | ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], |
955 | FREQ_QOS_MIN, policy->cpuinfo.min_freq); | |
956 | if (ret < 0) { | |
957 | dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); | |
958 | goto free_cpudata1; | |
959 | } | |
960 | ||
961 | ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], | |
962 | FREQ_QOS_MAX, policy->cpuinfo.max_freq); | |
963 | if (ret < 0) { | |
964 | dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); | |
965 | goto free_cpudata2; | |
966 | } | |
967 | ||
febab20c WK |
968 | cpudata->max_limit_freq = max_freq; |
969 | cpudata->min_limit_freq = min_freq; | |
ec437d71 HR |
970 | |
971 | policy->driver_data = cpudata; | |
972 | ||
41271016 | 973 | amd_pstate_boost_init(cpudata); |
abd61c08 PY |
974 | if (!current_pstate_driver->adjust_perf) |
975 | current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; | |
41271016 | 976 | |
ec437d71 HR |
977 | return 0; |
978 | ||
41271016 HR |
979 | free_cpudata2: |
980 | freq_qos_remove_request(&cpudata->req[0]); | |
981 | free_cpudata1: | |
ec437d71 HR |
982 | kfree(cpudata); |
983 | return ret; | |
984 | } | |
985 | ||
986 | static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) | |
987 | { | |
4f59540c | 988 | struct amd_cpudata *cpudata = policy->driver_data; |
ec437d71 | 989 | |
41271016 HR |
990 | freq_qos_remove_request(&cpudata->req[1]); |
991 | freq_qos_remove_request(&cpudata->req[0]); | |
4badf2eb | 992 | policy->fast_switch_possible = false; |
ec437d71 HR |
993 | kfree(cpudata); |
994 | ||
995 | return 0; | |
996 | } | |
997 | ||
b376471f JS |
998 | static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) |
999 | { | |
1000 | int ret; | |
1001 | ||
1002 | ret = amd_pstate_enable(true); | |
1003 | if (ret) | |
1004 | pr_err("failed to enable amd-pstate during resume, return %d\n", ret); | |
1005 | ||
1006 | return ret; | |
1007 | } | |
1008 | ||
1009 | static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) | |
1010 | { | |
1011 | int ret; | |
1012 | ||
1013 | ret = amd_pstate_enable(false); | |
1014 | if (ret) | |
1015 | pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); | |
1016 | ||
1017 | return ret; | |
1018 | } | |
1019 | ||
ec4e3326 HR |
1020 | /* Sysfs attributes */ |
1021 | ||
1022 | /* | |
1023 | * This frequency is to indicate the maximum hardware frequency. | |
1024 | * If boost is not active but supported, the frequency will be larger than the | |
1025 | * one in cpuinfo. | |
1026 | */ | |
1027 | static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, | |
1028 | char *buf) | |
1029 | { | |
1030 | int max_freq; | |
4f59540c | 1031 | struct amd_cpudata *cpudata = policy->driver_data; |
ec4e3326 | 1032 | |
3cbbe887 | 1033 | max_freq = READ_ONCE(cpudata->max_freq); |
ec4e3326 HR |
1034 | if (max_freq < 0) |
1035 | return max_freq; | |
1036 | ||
3ec32b6d | 1037 | return sysfs_emit(buf, "%u\n", max_freq); |
ec4e3326 HR |
1038 | } |
1039 | ||
1040 | static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, | |
1041 | char *buf) | |
1042 | { | |
1043 | int freq; | |
4f59540c | 1044 | struct amd_cpudata *cpudata = policy->driver_data; |
ec4e3326 | 1045 | |
3cbbe887 | 1046 | freq = READ_ONCE(cpudata->lowest_nonlinear_freq); |
ec4e3326 HR |
1047 | if (freq < 0) |
1048 | return freq; | |
1049 | ||
3ec32b6d | 1050 | return sysfs_emit(buf, "%u\n", freq); |
ec4e3326 HR |
1051 | } |
1052 | ||
3ad7fde1 HR |
1053 | /* |
1054 | * In some of ASICs, the highest_perf is not the one in the _CPC table, so we | |
1055 | * need to expose it to sysfs. | |
1056 | */ | |
1057 | static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, | |
1058 | char *buf) | |
1059 | { | |
1060 | u32 perf; | |
1061 | struct amd_cpudata *cpudata = policy->driver_data; | |
1062 | ||
1063 | perf = READ_ONCE(cpudata->highest_perf); | |
1064 | ||
3ec32b6d | 1065 | return sysfs_emit(buf, "%u\n", perf); |
3ad7fde1 HR |
1066 | } |
1067 | ||
e571a5e2 ML |
1068 | static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy, |
1069 | char *buf) | |
1070 | { | |
1071 | u32 perf; | |
1072 | struct amd_cpudata *cpudata = policy->driver_data; | |
1073 | ||
1074 | perf = READ_ONCE(cpudata->prefcore_ranking); | |
1075 | ||
1076 | return sysfs_emit(buf, "%u\n", perf); | |
1077 | } | |
1078 | ||
f3a05239 ML |
1079 | static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, |
1080 | char *buf) | |
1081 | { | |
1082 | bool hw_prefcore; | |
1083 | struct amd_cpudata *cpudata = policy->driver_data; | |
1084 | ||
1085 | hw_prefcore = READ_ONCE(cpudata->hw_prefcore); | |
1086 | ||
1087 | return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore)); | |
1088 | } | |
1089 | ||
ffa5096a PY |
1090 | static ssize_t show_energy_performance_available_preferences( |
1091 | struct cpufreq_policy *policy, char *buf) | |
1092 | { | |
1093 | int i = 0; | |
1094 | int offset = 0; | |
142c169b AJ |
1095 | struct amd_cpudata *cpudata = policy->driver_data; |
1096 | ||
1097 | if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) | |
1098 | return sysfs_emit_at(buf, offset, "%s\n", | |
1099 | energy_perf_strings[EPP_INDEX_PERFORMANCE]); | |
ffa5096a PY |
1100 | |
1101 | while (energy_perf_strings[i] != NULL) | |
1102 | offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); | |
1103 | ||
142c169b | 1104 | offset += sysfs_emit_at(buf, offset, "\n"); |
ffa5096a PY |
1105 | |
1106 | return offset; | |
1107 | } | |
1108 | ||
1109 | static ssize_t store_energy_performance_preference( | |
1110 | struct cpufreq_policy *policy, const char *buf, size_t count) | |
1111 | { | |
1112 | struct amd_cpudata *cpudata = policy->driver_data; | |
1113 | char str_preference[21]; | |
1114 | ssize_t ret; | |
1115 | ||
1116 | ret = sscanf(buf, "%20s", str_preference); | |
1117 | if (ret != 1) | |
1118 | return -EINVAL; | |
1119 | ||
1120 | ret = match_string(energy_perf_strings, -1, str_preference); | |
1121 | if (ret < 0) | |
1122 | return -EINVAL; | |
1123 | ||
1124 | mutex_lock(&amd_pstate_limits_lock); | |
1125 | ret = amd_pstate_set_energy_pref_index(cpudata, ret); | |
1126 | mutex_unlock(&amd_pstate_limits_lock); | |
1127 | ||
1128 | return ret ?: count; | |
1129 | } | |
1130 | ||
1131 | static ssize_t show_energy_performance_preference( | |
1132 | struct cpufreq_policy *policy, char *buf) | |
1133 | { | |
1134 | struct amd_cpudata *cpudata = policy->driver_data; | |
1135 | int preference; | |
1136 | ||
1137 | preference = amd_pstate_get_energy_pref_index(cpudata); | |
1138 | if (preference < 0) | |
1139 | return preference; | |
1140 | ||
1141 | return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); | |
1142 | } | |
1143 | ||
3ca7bc81 WK |
1144 | static void amd_pstate_driver_cleanup(void) |
1145 | { | |
1146 | amd_pstate_enable(false); | |
1147 | cppc_state = AMD_PSTATE_DISABLE; | |
1148 | current_pstate_driver = NULL; | |
1149 | } | |
1150 | ||
1151 | static int amd_pstate_register_driver(int mode) | |
1152 | { | |
1153 | int ret; | |
1154 | ||
1155 | if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED) | |
1156 | current_pstate_driver = &amd_pstate_driver; | |
1157 | else if (mode == AMD_PSTATE_ACTIVE) | |
1158 | current_pstate_driver = &amd_pstate_epp_driver; | |
1159 | else | |
1160 | return -EINVAL; | |
1161 | ||
1162 | cppc_state = mode; | |
1163 | ret = cpufreq_register_driver(current_pstate_driver); | |
1164 | if (ret) { | |
1165 | amd_pstate_driver_cleanup(); | |
1166 | return ret; | |
1167 | } | |
1168 | return 0; | |
1169 | } | |
1170 | ||
1171 | static int amd_pstate_unregister_driver(int dummy) | |
1172 | { | |
1173 | cpufreq_unregister_driver(current_pstate_driver); | |
1174 | amd_pstate_driver_cleanup(); | |
1175 | return 0; | |
1176 | } | |
1177 | ||
1178 | static int amd_pstate_change_mode_without_dvr_change(int mode) | |
1179 | { | |
1180 | int cpu = 0; | |
1181 | ||
1182 | cppc_state = mode; | |
1183 | ||
1184 | if (boot_cpu_has(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) | |
1185 | return 0; | |
1186 | ||
1187 | for_each_present_cpu(cpu) { | |
1188 | cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); | |
1189 | } | |
1190 | ||
1191 | return 0; | |
1192 | } | |
1193 | ||
1194 | static int amd_pstate_change_driver_mode(int mode) | |
1195 | { | |
1196 | int ret; | |
1197 | ||
1198 | ret = amd_pstate_unregister_driver(0); | |
1199 | if (ret) | |
1200 | return ret; | |
1201 | ||
1202 | ret = amd_pstate_register_driver(mode); | |
1203 | if (ret) | |
1204 | return ret; | |
1205 | ||
1206 | return 0; | |
1207 | } | |
1208 | ||
11fa52fe | 1209 | static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = { |
3ca7bc81 WK |
1210 | [AMD_PSTATE_DISABLE] = { |
1211 | [AMD_PSTATE_DISABLE] = NULL, | |
1212 | [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver, | |
1213 | [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver, | |
1214 | [AMD_PSTATE_GUIDED] = amd_pstate_register_driver, | |
1215 | }, | |
1216 | [AMD_PSTATE_PASSIVE] = { | |
1217 | [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, | |
1218 | [AMD_PSTATE_PASSIVE] = NULL, | |
1219 | [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, | |
1220 | [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change, | |
1221 | }, | |
1222 | [AMD_PSTATE_ACTIVE] = { | |
1223 | [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, | |
1224 | [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode, | |
1225 | [AMD_PSTATE_ACTIVE] = NULL, | |
1226 | [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode, | |
1227 | }, | |
1228 | [AMD_PSTATE_GUIDED] = { | |
1229 | [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver, | |
1230 | [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change, | |
1231 | [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode, | |
1232 | [AMD_PSTATE_GUIDED] = NULL, | |
1233 | }, | |
1234 | }; | |
1235 | ||
abd61c08 PY |
1236 | static ssize_t amd_pstate_show_status(char *buf) |
1237 | { | |
1238 | if (!current_pstate_driver) | |
1239 | return sysfs_emit(buf, "disable\n"); | |
1240 | ||
1241 | return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); | |
1242 | } | |
1243 | ||
abd61c08 PY |
1244 | static int amd_pstate_update_status(const char *buf, size_t size) |
1245 | { | |
abd61c08 PY |
1246 | int mode_idx; |
1247 | ||
3ca7bc81 | 1248 | if (size > strlen("passive") || size < strlen("active")) |
abd61c08 | 1249 | return -EINVAL; |
abd61c08 | 1250 | |
3ca7bc81 | 1251 | mode_idx = get_mode_idx_from_str(buf, size); |
abd61c08 | 1252 | |
3ca7bc81 WK |
1253 | if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) |
1254 | return -EINVAL; | |
abd61c08 | 1255 | |
3ca7bc81 WK |
1256 | if (mode_state_machine[cppc_state][mode_idx]) |
1257 | return mode_state_machine[cppc_state][mode_idx](mode_idx); | |
abd61c08 | 1258 | |
3ca7bc81 | 1259 | return 0; |
abd61c08 PY |
1260 | } |
1261 | ||
5e720f8c TW |
1262 | static ssize_t status_show(struct device *dev, |
1263 | struct device_attribute *attr, char *buf) | |
abd61c08 PY |
1264 | { |
1265 | ssize_t ret; | |
1266 | ||
1267 | mutex_lock(&amd_pstate_driver_lock); | |
1268 | ret = amd_pstate_show_status(buf); | |
1269 | mutex_unlock(&amd_pstate_driver_lock); | |
1270 | ||
1271 | return ret; | |
1272 | } | |
1273 | ||
5e720f8c | 1274 | static ssize_t status_store(struct device *a, struct device_attribute *b, |
abd61c08 PY |
1275 | const char *buf, size_t count) |
1276 | { | |
1277 | char *p = memchr(buf, '\n', count); | |
1278 | int ret; | |
1279 | ||
1280 | mutex_lock(&amd_pstate_driver_lock); | |
1281 | ret = amd_pstate_update_status(buf, p ? p - buf : count); | |
1282 | mutex_unlock(&amd_pstate_driver_lock); | |
1283 | ||
1284 | return ret < 0 ? ret : count; | |
1285 | } | |
1286 | ||
f3a05239 ML |
1287 | static ssize_t prefcore_show(struct device *dev, |
1288 | struct device_attribute *attr, char *buf) | |
1289 | { | |
1290 | return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); | |
1291 | } | |
1292 | ||
ec4e3326 HR |
1293 | cpufreq_freq_attr_ro(amd_pstate_max_freq); |
1294 | cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); | |
1295 | ||
3ad7fde1 | 1296 | cpufreq_freq_attr_ro(amd_pstate_highest_perf); |
e571a5e2 | 1297 | cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); |
f3a05239 | 1298 | cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); |
ffa5096a PY |
1299 | cpufreq_freq_attr_rw(energy_performance_preference); |
1300 | cpufreq_freq_attr_ro(energy_performance_available_preferences); | |
5e720f8c | 1301 | static DEVICE_ATTR_RW(status); |
f3a05239 | 1302 | static DEVICE_ATTR_RO(prefcore); |
3ad7fde1 | 1303 | |
ec4e3326 HR |
1304 | static struct freq_attr *amd_pstate_attr[] = { |
1305 | &amd_pstate_max_freq, | |
1306 | &amd_pstate_lowest_nonlinear_freq, | |
3ad7fde1 | 1307 | &amd_pstate_highest_perf, |
e571a5e2 | 1308 | &amd_pstate_prefcore_ranking, |
f3a05239 | 1309 | &amd_pstate_hw_prefcore, |
ec4e3326 HR |
1310 | NULL, |
1311 | }; | |
1312 | ||
ffa5096a PY |
1313 | static struct freq_attr *amd_pstate_epp_attr[] = { |
1314 | &amd_pstate_max_freq, | |
1315 | &amd_pstate_lowest_nonlinear_freq, | |
1316 | &amd_pstate_highest_perf, | |
e571a5e2 | 1317 | &amd_pstate_prefcore_ranking, |
f3a05239 | 1318 | &amd_pstate_hw_prefcore, |
ffa5096a PY |
1319 | &energy_performance_preference, |
1320 | &energy_performance_available_preferences, | |
1321 | NULL, | |
1322 | }; | |
1323 | ||
abd61c08 | 1324 | static struct attribute *pstate_global_attributes[] = { |
5e720f8c | 1325 | &dev_attr_status.attr, |
f3a05239 | 1326 | &dev_attr_prefcore.attr, |
abd61c08 PY |
1327 | NULL |
1328 | }; | |
1329 | ||
1330 | static const struct attribute_group amd_pstate_global_attr_group = { | |
3666062b | 1331 | .name = "amd_pstate", |
abd61c08 PY |
1332 | .attrs = pstate_global_attributes, |
1333 | }; | |
1334 | ||
32f80b9a ML |
1335 | static bool amd_pstate_acpi_pm_profile_server(void) |
1336 | { | |
1337 | switch (acpi_gbl_FADT.preferred_profile) { | |
1338 | case PM_ENTERPRISE_SERVER: | |
1339 | case PM_SOHO_SERVER: | |
1340 | case PM_PERFORMANCE_SERVER: | |
1341 | return true; | |
1342 | } | |
1343 | return false; | |
1344 | } | |
1345 | ||
1346 | static bool amd_pstate_acpi_pm_profile_undefined(void) | |
1347 | { | |
1348 | if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED) | |
1349 | return true; | |
1350 | if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES) | |
1351 | return true; | |
1352 | return false; | |
1353 | } | |
1354 | ||
ffa5096a PY |
1355 | static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) |
1356 | { | |
5c3fd1ed | 1357 | int min_freq, max_freq, nominal_freq, ret; |
ffa5096a PY |
1358 | struct amd_cpudata *cpudata; |
1359 | struct device *dev; | |
ffa5096a PY |
1360 | u64 value; |
1361 | ||
1362 | /* | |
1363 | * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, | |
1364 | * which is ideal for initialization process. | |
1365 | */ | |
1366 | amd_perf_ctl_reset(policy->cpu); | |
1367 | dev = get_cpu_device(policy->cpu); | |
1368 | if (!dev) | |
7cca9a98 | 1369 | return -ENODEV; |
ffa5096a PY |
1370 | |
1371 | cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); | |
1372 | if (!cpudata) | |
1373 | return -ENOMEM; | |
1374 | ||
1375 | cpudata->cpu = policy->cpu; | |
1376 | cpudata->epp_policy = 0; | |
1377 | ||
f3a05239 ML |
1378 | amd_pstate_init_prefcore(cpudata); |
1379 | ||
7cca9a98 AB |
1380 | ret = amd_pstate_init_perf(cpudata); |
1381 | if (ret) | |
ffa5096a PY |
1382 | goto free_cpudata1; |
1383 | ||
5547c0eb PY |
1384 | ret = amd_pstate_init_freq(cpudata); |
1385 | if (ret) | |
1386 | goto free_cpudata1; | |
1387 | ||
3cbbe887 GS |
1388 | min_freq = READ_ONCE(cpudata->min_freq); |
1389 | max_freq = READ_ONCE(cpudata->max_freq); | |
1390 | nominal_freq = READ_ONCE(cpudata->nominal_freq); | |
2ddb8a39 PY |
1391 | if (min_freq <= 0 || max_freq <= 0 || |
1392 | nominal_freq <= 0 || min_freq > max_freq) { | |
1393 | dev_err(dev, | |
1394 | "min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect, check _CPC in ACPI tables\n", | |
1395 | min_freq, max_freq, nominal_freq); | |
ffa5096a PY |
1396 | ret = -EINVAL; |
1397 | goto free_cpudata1; | |
1398 | } | |
1399 | ||
1400 | policy->cpuinfo.min_freq = min_freq; | |
1401 | policy->cpuinfo.max_freq = max_freq; | |
1402 | /* It will be updated by governor */ | |
1403 | policy->cur = policy->cpuinfo.min_freq; | |
1404 | ||
ffa5096a PY |
1405 | policy->driver_data = cpudata; |
1406 | ||
1407 | cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); | |
1408 | ||
1409 | policy->min = policy->cpuinfo.min_freq; | |
1410 | policy->max = policy->cpuinfo.max_freq; | |
1411 | ||
1412 | /* | |
32f80b9a | 1413 | * Set the policy to provide a valid fallback value in case |
ffa5096a PY |
1414 | * the default cpufreq governor is neither powersave nor performance. |
1415 | */ | |
32f80b9a ML |
1416 | if (amd_pstate_acpi_pm_profile_server() || |
1417 | amd_pstate_acpi_pm_profile_undefined()) | |
1418 | policy->policy = CPUFREQ_POLICY_PERFORMANCE; | |
1419 | else | |
1420 | policy->policy = CPUFREQ_POLICY_POWERSAVE; | |
ffa5096a PY |
1421 | |
1422 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
ffa5096a PY |
1423 | ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); |
1424 | if (ret) | |
1425 | return ret; | |
1426 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
1427 | ||
1428 | ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); | |
1429 | if (ret) | |
1430 | return ret; | |
1431 | WRITE_ONCE(cpudata->cppc_cap1_cached, value); | |
1432 | } | |
1433 | amd_pstate_boost_init(cpudata); | |
1434 | ||
1435 | return 0; | |
1436 | ||
1437 | free_cpudata1: | |
1438 | kfree(cpudata); | |
1439 | return ret; | |
1440 | } | |
1441 | ||
1442 | static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) | |
1443 | { | |
cea04f3d PM |
1444 | struct amd_cpudata *cpudata = policy->driver_data; |
1445 | ||
1446 | if (cpudata) { | |
1447 | kfree(cpudata); | |
1448 | policy->driver_data = NULL; | |
1449 | } | |
1450 | ||
ffa5096a | 1451 | pr_debug("CPU %d exiting\n", policy->cpu); |
ffa5096a PY |
1452 | return 0; |
1453 | } | |
1454 | ||
febab20c | 1455 | static void amd_pstate_epp_update_limit(struct cpufreq_policy *policy) |
ffa5096a | 1456 | { |
ffa5096a | 1457 | struct amd_cpudata *cpudata = policy->driver_data; |
febab20c | 1458 | u32 max_perf, min_perf, min_limit_perf, max_limit_perf; |
ffa5096a PY |
1459 | u64 value; |
1460 | s16 epp; | |
1461 | ||
1462 | max_perf = READ_ONCE(cpudata->highest_perf); | |
1463 | min_perf = READ_ONCE(cpudata->lowest_perf); | |
febab20c WK |
1464 | max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq); |
1465 | min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq); | |
1466 | ||
8164f743 ML |
1467 | if (min_limit_perf < min_perf) |
1468 | min_limit_perf = min_perf; | |
1469 | ||
1470 | if (max_limit_perf < min_limit_perf) | |
1471 | max_limit_perf = min_limit_perf; | |
1472 | ||
22fb4f04 ML |
1473 | WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf); |
1474 | WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf); | |
1475 | ||
febab20c WK |
1476 | max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf, |
1477 | cpudata->max_limit_perf); | |
1478 | min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf, | |
1479 | cpudata->max_limit_perf); | |
ffa5096a PY |
1480 | value = READ_ONCE(cpudata->cppc_req_cached); |
1481 | ||
1482 | if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) | |
1483 | min_perf = max_perf; | |
1484 | ||
1485 | /* Initial min/max values for CPPC Performance Controls Register */ | |
1486 | value &= ~AMD_CPPC_MIN_PERF(~0L); | |
1487 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
1488 | ||
1489 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
1490 | value |= AMD_CPPC_MAX_PERF(max_perf); | |
1491 | ||
1492 | /* CPPC EPP feature require to set zero to the desire perf bit */ | |
1493 | value &= ~AMD_CPPC_DES_PERF(~0L); | |
1494 | value |= AMD_CPPC_DES_PERF(0); | |
1495 | ||
ffa5096a PY |
1496 | cpudata->epp_policy = cpudata->policy; |
1497 | ||
6e9d1212 WK |
1498 | /* Get BIOS pre-defined epp value */ |
1499 | epp = amd_pstate_get_epp(cpudata, value); | |
1500 | if (epp < 0) { | |
1501 | /** | |
1502 | * This return value can only be negative for shared_memory | |
1503 | * systems where EPP register read/write not supported. | |
1504 | */ | |
febab20c | 1505 | return; |
ffa5096a | 1506 | } |
6e9d1212 WK |
1507 | |
1508 | if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) | |
1509 | epp = 0; | |
1510 | ||
ffa5096a PY |
1511 | /* Set initial EPP value */ |
1512 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1513 | value &= ~GENMASK_ULL(31, 24); | |
1514 | value |= (u64)epp << 24; | |
1515 | } | |
1516 | ||
6e9d1212 | 1517 | WRITE_ONCE(cpudata->cppc_req_cached, value); |
7cca9a98 | 1518 | amd_pstate_set_epp(cpudata, epp); |
ffa5096a PY |
1519 | } |
1520 | ||
1521 | static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) | |
1522 | { | |
1523 | struct amd_cpudata *cpudata = policy->driver_data; | |
1524 | ||
1525 | if (!policy->cpuinfo.max_freq) | |
1526 | return -ENODEV; | |
1527 | ||
1528 | pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", | |
1529 | policy->cpuinfo.max_freq, policy->max); | |
1530 | ||
1531 | cpudata->policy = policy->policy; | |
1532 | ||
febab20c | 1533 | amd_pstate_epp_update_limit(policy); |
ffa5096a PY |
1534 | |
1535 | return 0; | |
1536 | } | |
1537 | ||
d4da12f8 PY |
1538 | static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) |
1539 | { | |
1540 | struct cppc_perf_ctrls perf_ctrls; | |
1541 | u64 value, max_perf; | |
1542 | int ret; | |
1543 | ||
1544 | ret = amd_pstate_enable(true); | |
1545 | if (ret) | |
1546 | pr_err("failed to enable amd pstate during resume, return %d\n", ret); | |
1547 | ||
1548 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1549 | max_perf = READ_ONCE(cpudata->highest_perf); | |
1550 | ||
1551 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1552 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
1553 | } else { | |
1554 | perf_ctrls.max_perf = max_perf; | |
1555 | perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); | |
1556 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
1557 | } | |
1558 | } | |
1559 | ||
1560 | static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) | |
1561 | { | |
1562 | struct amd_cpudata *cpudata = policy->driver_data; | |
1563 | ||
1564 | pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); | |
1565 | ||
1566 | if (cppc_state == AMD_PSTATE_ACTIVE) { | |
1567 | amd_pstate_epp_reenable(cpudata); | |
1568 | cpudata->suspended = false; | |
1569 | } | |
1570 | ||
1571 | return 0; | |
1572 | } | |
1573 | ||
1574 | static void amd_pstate_epp_offline(struct cpufreq_policy *policy) | |
1575 | { | |
1576 | struct amd_cpudata *cpudata = policy->driver_data; | |
1577 | struct cppc_perf_ctrls perf_ctrls; | |
1578 | int min_perf; | |
1579 | u64 value; | |
1580 | ||
1581 | min_perf = READ_ONCE(cpudata->lowest_perf); | |
1582 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1583 | ||
1584 | mutex_lock(&amd_pstate_limits_lock); | |
1585 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1586 | cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; | |
1587 | ||
1588 | /* Set max perf same as min perf */ | |
1589 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
1590 | value |= AMD_CPPC_MAX_PERF(min_perf); | |
1591 | value &= ~AMD_CPPC_MIN_PERF(~0L); | |
1592 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
1593 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
1594 | } else { | |
1595 | perf_ctrls.desired_perf = 0; | |
1596 | perf_ctrls.max_perf = min_perf; | |
1597 | perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); | |
1598 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
1599 | } | |
1600 | mutex_unlock(&amd_pstate_limits_lock); | |
1601 | } | |
1602 | ||
1603 | static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) | |
1604 | { | |
1605 | struct amd_cpudata *cpudata = policy->driver_data; | |
1606 | ||
1607 | pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); | |
1608 | ||
1609 | if (cpudata->suspended) | |
1610 | return 0; | |
1611 | ||
1612 | if (cppc_state == AMD_PSTATE_ACTIVE) | |
1613 | amd_pstate_epp_offline(policy); | |
1614 | ||
1615 | return 0; | |
1616 | } | |
1617 | ||
ffa5096a PY |
1618 | static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) |
1619 | { | |
1620 | cpufreq_verify_within_cpu_limits(policy); | |
1621 | pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); | |
1622 | return 0; | |
1623 | } | |
1624 | ||
50ddd2f7 PY |
1625 | static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) |
1626 | { | |
1627 | struct amd_cpudata *cpudata = policy->driver_data; | |
1628 | int ret; | |
1629 | ||
1630 | /* avoid suspending when EPP is not enabled */ | |
1631 | if (cppc_state != AMD_PSTATE_ACTIVE) | |
1632 | return 0; | |
1633 | ||
1634 | /* set this flag to avoid setting core offline*/ | |
1635 | cpudata->suspended = true; | |
1636 | ||
1637 | /* disable CPPC in lowlevel firmware */ | |
1638 | ret = amd_pstate_enable(false); | |
1639 | if (ret) | |
1640 | pr_err("failed to suspend, return %d\n", ret); | |
1641 | ||
1642 | return 0; | |
1643 | } | |
1644 | ||
1645 | static int amd_pstate_epp_resume(struct cpufreq_policy *policy) | |
1646 | { | |
1647 | struct amd_cpudata *cpudata = policy->driver_data; | |
1648 | ||
1649 | if (cpudata->suspended) { | |
1650 | mutex_lock(&amd_pstate_limits_lock); | |
1651 | ||
1652 | /* enable amd pstate from suspend state*/ | |
1653 | amd_pstate_epp_reenable(cpudata); | |
1654 | ||
1655 | mutex_unlock(&amd_pstate_limits_lock); | |
1656 | ||
1657 | cpudata->suspended = false; | |
1658 | } | |
1659 | ||
1660 | return 0; | |
1661 | } | |
1662 | ||
ec437d71 HR |
1663 | static struct cpufreq_driver amd_pstate_driver = { |
1664 | .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, | |
1665 | .verify = amd_pstate_verify, | |
1666 | .target = amd_pstate_target, | |
4badf2eb | 1667 | .fast_switch = amd_pstate_fast_switch, |
ec437d71 HR |
1668 | .init = amd_pstate_cpu_init, |
1669 | .exit = amd_pstate_cpu_exit, | |
b376471f JS |
1670 | .suspend = amd_pstate_cpu_suspend, |
1671 | .resume = amd_pstate_cpu_resume, | |
41271016 | 1672 | .set_boost = amd_pstate_set_boost, |
e571a5e2 | 1673 | .update_limits = amd_pstate_update_limits, |
ec437d71 | 1674 | .name = "amd-pstate", |
d8bee41d | 1675 | .attr = amd_pstate_attr, |
ec437d71 HR |
1676 | }; |
1677 | ||
ffa5096a PY |
1678 | static struct cpufreq_driver amd_pstate_epp_driver = { |
1679 | .flags = CPUFREQ_CONST_LOOPS, | |
1680 | .verify = amd_pstate_epp_verify_policy, | |
1681 | .setpolicy = amd_pstate_epp_set_policy, | |
1682 | .init = amd_pstate_epp_cpu_init, | |
1683 | .exit = amd_pstate_epp_cpu_exit, | |
d4da12f8 PY |
1684 | .offline = amd_pstate_epp_cpu_offline, |
1685 | .online = amd_pstate_epp_cpu_online, | |
50ddd2f7 PY |
1686 | .suspend = amd_pstate_epp_suspend, |
1687 | .resume = amd_pstate_epp_resume, | |
e571a5e2 | 1688 | .update_limits = amd_pstate_update_limits, |
f4aad639 | 1689 | .name = "amd-pstate-epp", |
ffa5096a PY |
1690 | .attr = amd_pstate_epp_attr, |
1691 | }; | |
1692 | ||
c88ad30e ML |
1693 | static int __init amd_pstate_set_driver(int mode_idx) |
1694 | { | |
1695 | if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { | |
1696 | cppc_state = mode_idx; | |
1697 | if (cppc_state == AMD_PSTATE_DISABLE) | |
1698 | pr_info("driver is explicitly disabled\n"); | |
1699 | ||
1700 | if (cppc_state == AMD_PSTATE_ACTIVE) | |
1701 | current_pstate_driver = &amd_pstate_epp_driver; | |
1702 | ||
1703 | if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED) | |
1704 | current_pstate_driver = &amd_pstate_driver; | |
1705 | ||
1706 | return 0; | |
1707 | } | |
1708 | ||
1709 | return -EINVAL; | |
1710 | } | |
1711 | ||
ec437d71 HR |
1712 | static int __init amd_pstate_init(void) |
1713 | { | |
3666062b | 1714 | struct device *dev_root; |
ec437d71 HR |
1715 | int ret; |
1716 | ||
1717 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) | |
1718 | return -ENODEV; | |
1719 | ||
1720 | if (!acpi_cpc_valid()) { | |
a2a9d185 | 1721 | pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); |
ec437d71 HR |
1722 | return -ENODEV; |
1723 | } | |
1724 | ||
1725 | /* don't keep reloading if cpufreq_driver exists */ | |
1726 | if (cpufreq_get_current_driver()) | |
1727 | return -EEXIST; | |
1728 | ||
eb8b6c36 PY |
1729 | quirks = NULL; |
1730 | ||
1731 | /* check if this machine need CPPC quirks */ | |
1732 | dmi_check_system(amd_pstate_quirks_table); | |
1733 | ||
c88ad30e ML |
1734 | switch (cppc_state) { |
1735 | case AMD_PSTATE_UNDEFINED: | |
1736 | /* Disable on the following configs by default: | |
1737 | * 1. Undefined platforms | |
1738 | * 2. Server platforms | |
1739 | * 3. Shared memory designs | |
1740 | */ | |
1741 | if (amd_pstate_acpi_pm_profile_undefined() || | |
1742 | amd_pstate_acpi_pm_profile_server() || | |
1743 | !boot_cpu_has(X86_FEATURE_CPPC)) { | |
1744 | pr_info("driver load is disabled, boot with specific mode to enable this\n"); | |
1745 | return -ENODEV; | |
1746 | } | |
1747 | ret = amd_pstate_set_driver(CONFIG_X86_AMD_PSTATE_DEFAULT_MODE); | |
1748 | if (ret) | |
1749 | return ret; | |
1750 | break; | |
1751 | case AMD_PSTATE_DISABLE: | |
1752 | return -ENODEV; | |
1753 | case AMD_PSTATE_PASSIVE: | |
1754 | case AMD_PSTATE_ACTIVE: | |
1755 | case AMD_PSTATE_GUIDED: | |
1756 | break; | |
1757 | default: | |
1758 | return -EINVAL; | |
1759 | } | |
1760 | ||
ec437d71 | 1761 | /* capability check */ |
e059c184 HR |
1762 | if (boot_cpu_has(X86_FEATURE_CPPC)) { |
1763 | pr_debug("AMD CPPC MSR based functionality is supported\n"); | |
2dd6d0eb | 1764 | if (cppc_state != AMD_PSTATE_ACTIVE) |
ffa5096a | 1765 | current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; |
202e683d PY |
1766 | } else { |
1767 | pr_debug("AMD CPPC shared memory based functionality is supported\n"); | |
e059c184 HR |
1768 | static_call_update(amd_pstate_enable, cppc_enable); |
1769 | static_call_update(amd_pstate_init_perf, cppc_init_perf); | |
1770 | static_call_update(amd_pstate_update_perf, cppc_update_perf); | |
ec437d71 HR |
1771 | } |
1772 | ||
1773 | /* enable amd pstate feature */ | |
1774 | ret = amd_pstate_enable(true); | |
1775 | if (ret) { | |
ffa5096a | 1776 | pr_err("failed to enable with return %d\n", ret); |
ec437d71 HR |
1777 | return ret; |
1778 | } | |
1779 | ||
ffa5096a | 1780 | ret = cpufreq_register_driver(current_pstate_driver); |
ec437d71 | 1781 | if (ret) |
ffa5096a | 1782 | pr_err("failed to register with return %d\n", ret); |
ec437d71 | 1783 | |
3666062b GKH |
1784 | dev_root = bus_get_dev_root(&cpu_subsys); |
1785 | if (dev_root) { | |
1786 | ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group); | |
1787 | put_device(dev_root); | |
1788 | if (ret) { | |
1789 | pr_err("sysfs attribute export failed with error %d.\n", ret); | |
1790 | goto global_attr_free; | |
1791 | } | |
abd61c08 PY |
1792 | } |
1793 | ||
1794 | return ret; | |
1795 | ||
1796 | global_attr_free: | |
abd61c08 | 1797 | cpufreq_unregister_driver(current_pstate_driver); |
ec437d71 HR |
1798 | return ret; |
1799 | } | |
456ca88d | 1800 | device_initcall(amd_pstate_init); |
ec437d71 | 1801 | |
202e683d PY |
1802 | static int __init amd_pstate_param(char *str) |
1803 | { | |
36c5014e WK |
1804 | size_t size; |
1805 | int mode_idx; | |
1806 | ||
202e683d PY |
1807 | if (!str) |
1808 | return -EINVAL; | |
1809 | ||
36c5014e WK |
1810 | size = strlen(str); |
1811 | mode_idx = get_mode_idx_from_str(str, size); | |
202e683d | 1812 | |
c88ad30e | 1813 | return amd_pstate_set_driver(mode_idx); |
202e683d | 1814 | } |
f3a05239 ML |
1815 | |
1816 | static int __init amd_prefcore_param(char *str) | |
1817 | { | |
1818 | if (!strcmp(str, "disable")) | |
1819 | amd_pstate_prefcore = false; | |
1820 | ||
1821 | return 0; | |
1822 | } | |
1823 | ||
202e683d | 1824 | early_param("amd_pstate", amd_pstate_param); |
f3a05239 | 1825 | early_param("amd_prefcore", amd_prefcore_param); |
202e683d | 1826 | |
ec437d71 HR |
1827 | MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); |
1828 | MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); |