Commit | Line | Data |
---|---|---|
ec437d71 HR |
1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* | |
3 | * amd-pstate.c - AMD Processor P-state Frequency Driver | |
4 | * | |
5 | * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved. | |
6 | * | |
7 | * Author: Huang Rui <ray.huang@amd.com> | |
8 | * | |
9 | * AMD P-State introduces a new CPU performance scaling design for AMD | |
10 | * processors using the ACPI Collaborative Performance and Power Control (CPPC) | |
11 | * feature which works with the AMD SMU firmware providing a finer grained | |
12 | * frequency control range. It is to replace the legacy ACPI P-States control, | |
13 | * allows a flexible, low-latency interface for the Linux kernel to directly | |
14 | * communicate the performance hints to hardware. | |
15 | * | |
16 | * AMD P-State is supported on recent AMD Zen base CPU series include some of | |
17 | * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD | |
18 | * P-State supported system. And there are two types of hardware implementations | |
19 | * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution. | |
20 | * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types. | |
21 | */ | |
22 | ||
23 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
24 | ||
25 | #include <linux/kernel.h> | |
26 | #include <linux/module.h> | |
27 | #include <linux/init.h> | |
28 | #include <linux/smp.h> | |
29 | #include <linux/sched.h> | |
30 | #include <linux/cpufreq.h> | |
31 | #include <linux/compiler.h> | |
32 | #include <linux/dmi.h> | |
33 | #include <linux/slab.h> | |
34 | #include <linux/acpi.h> | |
35 | #include <linux/io.h> | |
36 | #include <linux/delay.h> | |
37 | #include <linux/uaccess.h> | |
38 | #include <linux/static_call.h> | |
f1375ec1 | 39 | #include <linux/amd-pstate.h> |
ec437d71 HR |
40 | |
41 | #include <acpi/processor.h> | |
42 | #include <acpi/cppc_acpi.h> | |
43 | ||
44 | #include <asm/msr.h> | |
45 | #include <asm/processor.h> | |
46 | #include <asm/cpufeature.h> | |
47 | #include <asm/cpu_device_id.h> | |
60e10f89 | 48 | #include "amd-pstate-trace.h" |
ec437d71 | 49 | |
ca08e46d PY |
50 | #define AMD_PSTATE_TRANSITION_LATENCY 20000 |
51 | #define AMD_PSTATE_TRANSITION_DELAY 1000 | |
ec437d71 | 52 | |
e059c184 HR |
53 | /* |
54 | * TODO: We need more time to fine tune processors with shared memory solution | |
55 | * with community together. | |
56 | * | |
57 | * There are some performance drops on the CPU benchmarks which reports from | |
58 | * Suse. We are co-working with them to fine tune the shared memory solution. So | |
59 | * we disable it by default to go acpi-cpufreq on these processors and add a | |
60 | * module parameter to be able to enable it manually for debugging. | |
61 | */ | |
ffa5096a | 62 | static struct cpufreq_driver *current_pstate_driver; |
ec437d71 | 63 | static struct cpufreq_driver amd_pstate_driver; |
ffa5096a | 64 | static struct cpufreq_driver amd_pstate_epp_driver; |
36c5014e | 65 | static int cppc_state = AMD_PSTATE_DISABLE; |
abd61c08 | 66 | struct kobject *amd_pstate_kobj; |
36c5014e | 67 | |
ffa5096a PY |
68 | /* |
69 | * AMD Energy Preference Performance (EPP) | |
70 | * The EPP is used in the CCLK DPM controller to drive | |
71 | * the frequency that a core is going to operate during | |
72 | * short periods of activity. EPP values will be utilized for | |
73 | * different OS profiles (balanced, performance, power savings) | |
74 | * display strings corresponding to EPP index in the | |
75 | * energy_perf_strings[] | |
76 | * index String | |
77 | *------------------------------------- | |
78 | * 0 default | |
79 | * 1 performance | |
80 | * 2 balance_performance | |
81 | * 3 balance_power | |
82 | * 4 power | |
83 | */ | |
84 | enum energy_perf_value_index { | |
85 | EPP_INDEX_DEFAULT = 0, | |
86 | EPP_INDEX_PERFORMANCE, | |
87 | EPP_INDEX_BALANCE_PERFORMANCE, | |
88 | EPP_INDEX_BALANCE_POWERSAVE, | |
89 | EPP_INDEX_POWERSAVE, | |
90 | }; | |
91 | ||
92 | static const char * const energy_perf_strings[] = { | |
93 | [EPP_INDEX_DEFAULT] = "default", | |
94 | [EPP_INDEX_PERFORMANCE] = "performance", | |
95 | [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", | |
96 | [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", | |
97 | [EPP_INDEX_POWERSAVE] = "power", | |
98 | NULL | |
99 | }; | |
100 | ||
101 | static unsigned int epp_values[] = { | |
102 | [EPP_INDEX_DEFAULT] = 0, | |
103 | [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE, | |
104 | [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, | |
105 | [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, | |
106 | [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, | |
107 | }; | |
108 | ||
36c5014e WK |
109 | static inline int get_mode_idx_from_str(const char *str, size_t size) |
110 | { | |
111 | int i; | |
112 | ||
113 | for (i=0; i < AMD_PSTATE_MAX; i++) { | |
114 | if (!strncmp(str, amd_pstate_mode_string[i], size)) | |
115 | return i; | |
116 | } | |
117 | return -EINVAL; | |
118 | } | |
ec437d71 | 119 | |
ffa5096a PY |
120 | static DEFINE_MUTEX(amd_pstate_limits_lock); |
121 | static DEFINE_MUTEX(amd_pstate_driver_lock); | |
122 | ||
123 | static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached) | |
124 | { | |
125 | u64 epp; | |
126 | int ret; | |
127 | ||
128 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
129 | if (!cppc_req_cached) { | |
130 | epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, | |
131 | &cppc_req_cached); | |
132 | if (epp) | |
133 | return epp; | |
134 | } | |
135 | epp = (cppc_req_cached >> 24) & 0xFF; | |
136 | } else { | |
137 | ret = cppc_get_epp_perf(cpudata->cpu, &epp); | |
138 | if (ret < 0) { | |
139 | pr_debug("Could not retrieve energy perf value (%d)\n", ret); | |
140 | return -EIO; | |
141 | } | |
142 | } | |
143 | ||
144 | return (s16)(epp & 0xff); | |
145 | } | |
146 | ||
147 | static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata) | |
148 | { | |
149 | s16 epp; | |
150 | int index = -EINVAL; | |
151 | ||
152 | epp = amd_pstate_get_epp(cpudata, 0); | |
153 | if (epp < 0) | |
154 | return epp; | |
155 | ||
156 | switch (epp) { | |
157 | case AMD_CPPC_EPP_PERFORMANCE: | |
158 | index = EPP_INDEX_PERFORMANCE; | |
159 | break; | |
160 | case AMD_CPPC_EPP_BALANCE_PERFORMANCE: | |
161 | index = EPP_INDEX_BALANCE_PERFORMANCE; | |
162 | break; | |
163 | case AMD_CPPC_EPP_BALANCE_POWERSAVE: | |
164 | index = EPP_INDEX_BALANCE_POWERSAVE; | |
165 | break; | |
166 | case AMD_CPPC_EPP_POWERSAVE: | |
167 | index = EPP_INDEX_POWERSAVE; | |
168 | break; | |
169 | default: | |
170 | break; | |
171 | } | |
172 | ||
173 | return index; | |
174 | } | |
175 | ||
176 | static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp) | |
177 | { | |
178 | int ret; | |
179 | struct cppc_perf_ctrls perf_ctrls; | |
180 | ||
181 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
182 | u64 value = READ_ONCE(cpudata->cppc_req_cached); | |
183 | ||
184 | value &= ~GENMASK_ULL(31, 24); | |
185 | value |= (u64)epp << 24; | |
186 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
187 | ||
188 | ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
189 | if (!ret) | |
190 | cpudata->epp_cached = epp; | |
191 | } else { | |
192 | perf_ctrls.energy_perf = epp; | |
193 | ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1); | |
194 | if (ret) { | |
195 | pr_debug("failed to set energy perf value (%d)\n", ret); | |
196 | return ret; | |
197 | } | |
198 | cpudata->epp_cached = epp; | |
199 | } | |
200 | ||
201 | return ret; | |
202 | } | |
203 | ||
204 | static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata, | |
205 | int pref_index) | |
206 | { | |
207 | int epp = -EINVAL; | |
208 | int ret; | |
209 | ||
210 | if (!pref_index) { | |
211 | pr_debug("EPP pref_index is invalid\n"); | |
212 | return -EINVAL; | |
213 | } | |
214 | ||
215 | if (epp == -EINVAL) | |
216 | epp = epp_values[pref_index]; | |
217 | ||
218 | if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { | |
219 | pr_debug("EPP cannot be set under performance policy\n"); | |
220 | return -EBUSY; | |
221 | } | |
222 | ||
223 | ret = amd_pstate_set_epp(cpudata, epp); | |
224 | ||
225 | return ret; | |
226 | } | |
227 | ||
e059c184 | 228 | static inline int pstate_enable(bool enable) |
ec437d71 HR |
229 | { |
230 | return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable); | |
231 | } | |
232 | ||
e059c184 HR |
233 | static int cppc_enable(bool enable) |
234 | { | |
235 | int cpu, ret = 0; | |
ffa5096a | 236 | struct cppc_perf_ctrls perf_ctrls; |
e059c184 HR |
237 | |
238 | for_each_present_cpu(cpu) { | |
239 | ret = cppc_set_enable(cpu, enable); | |
240 | if (ret) | |
241 | return ret; | |
ffa5096a PY |
242 | |
243 | /* Enable autonomous mode for EPP */ | |
244 | if (cppc_state == AMD_PSTATE_ACTIVE) { | |
245 | /* Set desired perf as zero to allow EPP firmware control */ | |
246 | perf_ctrls.desired_perf = 0; | |
247 | ret = cppc_set_perf(cpu, &perf_ctrls); | |
248 | if (ret) | |
249 | return ret; | |
250 | } | |
e059c184 HR |
251 | } |
252 | ||
253 | return ret; | |
254 | } | |
255 | ||
256 | DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable); | |
257 | ||
258 | static inline int amd_pstate_enable(bool enable) | |
259 | { | |
260 | return static_call(amd_pstate_enable)(enable); | |
261 | } | |
262 | ||
263 | static int pstate_init_perf(struct amd_cpudata *cpudata) | |
ec437d71 HR |
264 | { |
265 | u64 cap1; | |
bedadcfb | 266 | u32 highest_perf; |
ec437d71 HR |
267 | |
268 | int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, | |
269 | &cap1); | |
270 | if (ret) | |
271 | return ret; | |
272 | ||
273 | /* | |
274 | * TODO: Introduce AMD specific power feature. | |
275 | * | |
276 | * CPPC entry doesn't indicate the highest performance in some ASICs. | |
277 | */ | |
bedadcfb PY |
278 | highest_perf = amd_get_highest_perf(); |
279 | if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1)) | |
280 | highest_perf = AMD_CPPC_HIGHEST_PERF(cap1); | |
281 | ||
282 | WRITE_ONCE(cpudata->highest_perf, highest_perf); | |
ec437d71 HR |
283 | |
284 | WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1)); | |
285 | WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1)); | |
286 | WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1)); | |
287 | ||
288 | return 0; | |
289 | } | |
290 | ||
e059c184 HR |
291 | static int cppc_init_perf(struct amd_cpudata *cpudata) |
292 | { | |
293 | struct cppc_perf_caps cppc_perf; | |
bedadcfb | 294 | u32 highest_perf; |
e059c184 HR |
295 | |
296 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
297 | if (ret) | |
298 | return ret; | |
299 | ||
bedadcfb PY |
300 | highest_perf = amd_get_highest_perf(); |
301 | if (highest_perf > cppc_perf.highest_perf) | |
302 | highest_perf = cppc_perf.highest_perf; | |
303 | ||
304 | WRITE_ONCE(cpudata->highest_perf, highest_perf); | |
e059c184 HR |
305 | |
306 | WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf); | |
307 | WRITE_ONCE(cpudata->lowest_nonlinear_perf, | |
308 | cppc_perf.lowest_nonlinear_perf); | |
309 | WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf); | |
310 | ||
311 | return 0; | |
312 | } | |
313 | ||
314 | DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf); | |
315 | ||
316 | static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata) | |
317 | { | |
318 | return static_call(amd_pstate_init_perf)(cpudata); | |
319 | } | |
320 | ||
321 | static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf, | |
322 | u32 des_perf, u32 max_perf, bool fast_switch) | |
ec437d71 HR |
323 | { |
324 | if (fast_switch) | |
325 | wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached)); | |
326 | else | |
327 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, | |
328 | READ_ONCE(cpudata->cppc_req_cached)); | |
329 | } | |
330 | ||
e059c184 HR |
331 | static void cppc_update_perf(struct amd_cpudata *cpudata, |
332 | u32 min_perf, u32 des_perf, | |
333 | u32 max_perf, bool fast_switch) | |
334 | { | |
335 | struct cppc_perf_ctrls perf_ctrls; | |
336 | ||
337 | perf_ctrls.max_perf = max_perf; | |
338 | perf_ctrls.min_perf = min_perf; | |
339 | perf_ctrls.desired_perf = des_perf; | |
340 | ||
341 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
342 | } | |
343 | ||
344 | DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf); | |
345 | ||
346 | static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata, | |
347 | u32 min_perf, u32 des_perf, | |
348 | u32 max_perf, bool fast_switch) | |
349 | { | |
350 | static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf, | |
351 | max_perf, fast_switch); | |
352 | } | |
353 | ||
23c296fb JS |
354 | static inline bool amd_pstate_sample(struct amd_cpudata *cpudata) |
355 | { | |
356 | u64 aperf, mperf, tsc; | |
357 | unsigned long flags; | |
358 | ||
359 | local_irq_save(flags); | |
360 | rdmsrl(MSR_IA32_APERF, aperf); | |
361 | rdmsrl(MSR_IA32_MPERF, mperf); | |
362 | tsc = rdtsc(); | |
363 | ||
364 | if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) { | |
365 | local_irq_restore(flags); | |
366 | return false; | |
367 | } | |
368 | ||
369 | local_irq_restore(flags); | |
370 | ||
371 | cpudata->cur.aperf = aperf; | |
372 | cpudata->cur.mperf = mperf; | |
373 | cpudata->cur.tsc = tsc; | |
374 | cpudata->cur.aperf -= cpudata->prev.aperf; | |
375 | cpudata->cur.mperf -= cpudata->prev.mperf; | |
376 | cpudata->cur.tsc -= cpudata->prev.tsc; | |
377 | ||
378 | cpudata->prev.aperf = aperf; | |
379 | cpudata->prev.mperf = mperf; | |
380 | cpudata->prev.tsc = tsc; | |
381 | ||
382 | cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf); | |
383 | ||
384 | return true; | |
385 | } | |
386 | ||
ec437d71 HR |
387 | static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf, |
388 | u32 des_perf, u32 max_perf, bool fast_switch) | |
389 | { | |
390 | u64 prev = READ_ONCE(cpudata->cppc_req_cached); | |
391 | u64 value = prev; | |
392 | ||
0e9a8638 | 393 | des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf); |
ec437d71 HR |
394 | value &= ~AMD_CPPC_MIN_PERF(~0L); |
395 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
396 | ||
397 | value &= ~AMD_CPPC_DES_PERF(~0L); | |
398 | value |= AMD_CPPC_DES_PERF(des_perf); | |
399 | ||
400 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
401 | value |= AMD_CPPC_MAX_PERF(max_perf); | |
402 | ||
23c296fb JS |
403 | if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) { |
404 | trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq, | |
405 | cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc, | |
406 | cpudata->cpu, (value != prev), fast_switch); | |
407 | } | |
60e10f89 | 408 | |
ec437d71 HR |
409 | if (value == prev) |
410 | return; | |
411 | ||
412 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
413 | ||
414 | amd_pstate_update_perf(cpudata, min_perf, des_perf, | |
415 | max_perf, fast_switch); | |
416 | } | |
417 | ||
418 | static int amd_pstate_verify(struct cpufreq_policy_data *policy) | |
419 | { | |
420 | cpufreq_verify_within_cpu_limits(policy); | |
421 | ||
422 | return 0; | |
423 | } | |
424 | ||
425 | static int amd_pstate_target(struct cpufreq_policy *policy, | |
426 | unsigned int target_freq, | |
427 | unsigned int relation) | |
428 | { | |
429 | struct cpufreq_freqs freqs; | |
430 | struct amd_cpudata *cpudata = policy->driver_data; | |
431 | unsigned long max_perf, min_perf, des_perf, cap_perf; | |
432 | ||
433 | if (!cpudata->max_freq) | |
434 | return -ENODEV; | |
435 | ||
436 | cap_perf = READ_ONCE(cpudata->highest_perf); | |
b185c505 | 437 | min_perf = READ_ONCE(cpudata->lowest_perf); |
ec437d71 HR |
438 | max_perf = cap_perf; |
439 | ||
440 | freqs.old = policy->cur; | |
441 | freqs.new = target_freq; | |
442 | ||
443 | des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf, | |
444 | cpudata->max_freq); | |
445 | ||
446 | cpufreq_freq_transition_begin(policy, &freqs); | |
447 | amd_pstate_update(cpudata, min_perf, des_perf, | |
448 | max_perf, false); | |
449 | cpufreq_freq_transition_end(policy, &freqs, false); | |
450 | ||
451 | return 0; | |
452 | } | |
453 | ||
1d215f03 HR |
454 | static void amd_pstate_adjust_perf(unsigned int cpu, |
455 | unsigned long _min_perf, | |
456 | unsigned long target_perf, | |
457 | unsigned long capacity) | |
458 | { | |
459 | unsigned long max_perf, min_perf, des_perf, | |
460 | cap_perf, lowest_nonlinear_perf; | |
461 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); | |
462 | struct amd_cpudata *cpudata = policy->driver_data; | |
463 | ||
464 | cap_perf = READ_ONCE(cpudata->highest_perf); | |
465 | lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf); | |
466 | ||
467 | des_perf = cap_perf; | |
468 | if (target_perf < capacity) | |
469 | des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity); | |
470 | ||
471 | min_perf = READ_ONCE(cpudata->highest_perf); | |
472 | if (_min_perf < capacity) | |
473 | min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity); | |
474 | ||
475 | if (min_perf < lowest_nonlinear_perf) | |
476 | min_perf = lowest_nonlinear_perf; | |
477 | ||
478 | max_perf = cap_perf; | |
479 | if (max_perf < min_perf) | |
480 | max_perf = min_perf; | |
481 | ||
1d215f03 | 482 | amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true); |
4f3085f8 | 483 | cpufreq_cpu_put(policy); |
1d215f03 HR |
484 | } |
485 | ||
ec437d71 HR |
486 | static int amd_get_min_freq(struct amd_cpudata *cpudata) |
487 | { | |
488 | struct cppc_perf_caps cppc_perf; | |
489 | ||
490 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
491 | if (ret) | |
492 | return ret; | |
493 | ||
494 | /* Switch to khz */ | |
495 | return cppc_perf.lowest_freq * 1000; | |
496 | } | |
497 | ||
498 | static int amd_get_max_freq(struct amd_cpudata *cpudata) | |
499 | { | |
500 | struct cppc_perf_caps cppc_perf; | |
501 | u32 max_perf, max_freq, nominal_freq, nominal_perf; | |
502 | u64 boost_ratio; | |
503 | ||
504 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
505 | if (ret) | |
506 | return ret; | |
507 | ||
508 | nominal_freq = cppc_perf.nominal_freq; | |
509 | nominal_perf = READ_ONCE(cpudata->nominal_perf); | |
510 | max_perf = READ_ONCE(cpudata->highest_perf); | |
511 | ||
512 | boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT, | |
513 | nominal_perf); | |
514 | ||
515 | max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT; | |
516 | ||
517 | /* Switch to khz */ | |
518 | return max_freq * 1000; | |
519 | } | |
520 | ||
521 | static int amd_get_nominal_freq(struct amd_cpudata *cpudata) | |
522 | { | |
523 | struct cppc_perf_caps cppc_perf; | |
524 | ||
525 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
526 | if (ret) | |
527 | return ret; | |
528 | ||
529 | /* Switch to khz */ | |
530 | return cppc_perf.nominal_freq * 1000; | |
531 | } | |
532 | ||
533 | static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata) | |
534 | { | |
535 | struct cppc_perf_caps cppc_perf; | |
536 | u32 lowest_nonlinear_freq, lowest_nonlinear_perf, | |
537 | nominal_freq, nominal_perf; | |
538 | u64 lowest_nonlinear_ratio; | |
539 | ||
540 | int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf); | |
541 | if (ret) | |
542 | return ret; | |
543 | ||
544 | nominal_freq = cppc_perf.nominal_freq; | |
545 | nominal_perf = READ_ONCE(cpudata->nominal_perf); | |
546 | ||
547 | lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf; | |
548 | ||
549 | lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT, | |
550 | nominal_perf); | |
551 | ||
552 | lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT; | |
553 | ||
554 | /* Switch to khz */ | |
555 | return lowest_nonlinear_freq * 1000; | |
556 | } | |
557 | ||
41271016 HR |
558 | static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state) |
559 | { | |
560 | struct amd_cpudata *cpudata = policy->driver_data; | |
561 | int ret; | |
562 | ||
563 | if (!cpudata->boost_supported) { | |
564 | pr_err("Boost mode is not supported by this processor or SBIOS\n"); | |
565 | return -EINVAL; | |
566 | } | |
567 | ||
568 | if (state) | |
569 | policy->cpuinfo.max_freq = cpudata->max_freq; | |
570 | else | |
571 | policy->cpuinfo.max_freq = cpudata->nominal_freq; | |
572 | ||
573 | policy->max = policy->cpuinfo.max_freq; | |
574 | ||
575 | ret = freq_qos_update_request(&cpudata->req[1], | |
576 | policy->cpuinfo.max_freq); | |
577 | if (ret < 0) | |
578 | return ret; | |
579 | ||
580 | return 0; | |
581 | } | |
582 | ||
583 | static void amd_pstate_boost_init(struct amd_cpudata *cpudata) | |
584 | { | |
585 | u32 highest_perf, nominal_perf; | |
586 | ||
587 | highest_perf = READ_ONCE(cpudata->highest_perf); | |
588 | nominal_perf = READ_ONCE(cpudata->nominal_perf); | |
589 | ||
590 | if (highest_perf <= nominal_perf) | |
591 | return; | |
592 | ||
593 | cpudata->boost_supported = true; | |
ffa5096a | 594 | current_pstate_driver->boost_enabled = true; |
41271016 HR |
595 | } |
596 | ||
919f4557 WK |
597 | static void amd_perf_ctl_reset(unsigned int cpu) |
598 | { | |
599 | wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0); | |
600 | } | |
601 | ||
ec437d71 HR |
602 | static int amd_pstate_cpu_init(struct cpufreq_policy *policy) |
603 | { | |
604 | int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; | |
605 | struct device *dev; | |
606 | struct amd_cpudata *cpudata; | |
607 | ||
919f4557 WK |
608 | /* |
609 | * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, | |
610 | * which is ideal for initialization process. | |
611 | */ | |
612 | amd_perf_ctl_reset(policy->cpu); | |
ec437d71 HR |
613 | dev = get_cpu_device(policy->cpu); |
614 | if (!dev) | |
615 | return -ENODEV; | |
616 | ||
617 | cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); | |
618 | if (!cpudata) | |
619 | return -ENOMEM; | |
620 | ||
621 | cpudata->cpu = policy->cpu; | |
622 | ||
623 | ret = amd_pstate_init_perf(cpudata); | |
624 | if (ret) | |
41271016 | 625 | goto free_cpudata1; |
ec437d71 HR |
626 | |
627 | min_freq = amd_get_min_freq(cpudata); | |
628 | max_freq = amd_get_max_freq(cpudata); | |
629 | nominal_freq = amd_get_nominal_freq(cpudata); | |
630 | lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); | |
631 | ||
632 | if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { | |
633 | dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", | |
634 | min_freq, max_freq); | |
635 | ret = -EINVAL; | |
41271016 | 636 | goto free_cpudata1; |
ec437d71 HR |
637 | } |
638 | ||
639 | policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY; | |
640 | policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY; | |
641 | ||
642 | policy->min = min_freq; | |
643 | policy->max = max_freq; | |
644 | ||
645 | policy->cpuinfo.min_freq = min_freq; | |
646 | policy->cpuinfo.max_freq = max_freq; | |
647 | ||
648 | /* It will be updated by governor */ | |
649 | policy->cur = policy->cpuinfo.min_freq; | |
650 | ||
e059c184 HR |
651 | if (boot_cpu_has(X86_FEATURE_CPPC)) |
652 | policy->fast_switch_possible = true; | |
1d215f03 | 653 | |
41271016 HR |
654 | ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0], |
655 | FREQ_QOS_MIN, policy->cpuinfo.min_freq); | |
656 | if (ret < 0) { | |
657 | dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret); | |
658 | goto free_cpudata1; | |
659 | } | |
660 | ||
661 | ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1], | |
662 | FREQ_QOS_MAX, policy->cpuinfo.max_freq); | |
663 | if (ret < 0) { | |
664 | dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret); | |
665 | goto free_cpudata2; | |
666 | } | |
667 | ||
ec437d71 HR |
668 | /* Initial processor data capability frequencies */ |
669 | cpudata->max_freq = max_freq; | |
670 | cpudata->min_freq = min_freq; | |
671 | cpudata->nominal_freq = nominal_freq; | |
672 | cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; | |
673 | ||
674 | policy->driver_data = cpudata; | |
675 | ||
41271016 | 676 | amd_pstate_boost_init(cpudata); |
abd61c08 PY |
677 | if (!current_pstate_driver->adjust_perf) |
678 | current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; | |
41271016 | 679 | |
ec437d71 HR |
680 | return 0; |
681 | ||
41271016 HR |
682 | free_cpudata2: |
683 | freq_qos_remove_request(&cpudata->req[0]); | |
684 | free_cpudata1: | |
ec437d71 HR |
685 | kfree(cpudata); |
686 | return ret; | |
687 | } | |
688 | ||
689 | static int amd_pstate_cpu_exit(struct cpufreq_policy *policy) | |
690 | { | |
4f59540c | 691 | struct amd_cpudata *cpudata = policy->driver_data; |
ec437d71 | 692 | |
41271016 HR |
693 | freq_qos_remove_request(&cpudata->req[1]); |
694 | freq_qos_remove_request(&cpudata->req[0]); | |
ec437d71 HR |
695 | kfree(cpudata); |
696 | ||
697 | return 0; | |
698 | } | |
699 | ||
b376471f JS |
700 | static int amd_pstate_cpu_resume(struct cpufreq_policy *policy) |
701 | { | |
702 | int ret; | |
703 | ||
704 | ret = amd_pstate_enable(true); | |
705 | if (ret) | |
706 | pr_err("failed to enable amd-pstate during resume, return %d\n", ret); | |
707 | ||
708 | return ret; | |
709 | } | |
710 | ||
711 | static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy) | |
712 | { | |
713 | int ret; | |
714 | ||
715 | ret = amd_pstate_enable(false); | |
716 | if (ret) | |
717 | pr_err("failed to disable amd-pstate during suspend, return %d\n", ret); | |
718 | ||
719 | return ret; | |
720 | } | |
721 | ||
ec4e3326 HR |
722 | /* Sysfs attributes */ |
723 | ||
724 | /* | |
725 | * This frequency is to indicate the maximum hardware frequency. | |
726 | * If boost is not active but supported, the frequency will be larger than the | |
727 | * one in cpuinfo. | |
728 | */ | |
729 | static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy, | |
730 | char *buf) | |
731 | { | |
732 | int max_freq; | |
4f59540c | 733 | struct amd_cpudata *cpudata = policy->driver_data; |
ec4e3326 HR |
734 | |
735 | max_freq = amd_get_max_freq(cpudata); | |
736 | if (max_freq < 0) | |
737 | return max_freq; | |
738 | ||
3ec32b6d | 739 | return sysfs_emit(buf, "%u\n", max_freq); |
ec4e3326 HR |
740 | } |
741 | ||
742 | static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy, | |
743 | char *buf) | |
744 | { | |
745 | int freq; | |
4f59540c | 746 | struct amd_cpudata *cpudata = policy->driver_data; |
ec4e3326 HR |
747 | |
748 | freq = amd_get_lowest_nonlinear_freq(cpudata); | |
749 | if (freq < 0) | |
750 | return freq; | |
751 | ||
3ec32b6d | 752 | return sysfs_emit(buf, "%u\n", freq); |
ec4e3326 HR |
753 | } |
754 | ||
3ad7fde1 HR |
755 | /* |
756 | * In some of ASICs, the highest_perf is not the one in the _CPC table, so we | |
757 | * need to expose it to sysfs. | |
758 | */ | |
759 | static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy, | |
760 | char *buf) | |
761 | { | |
762 | u32 perf; | |
763 | struct amd_cpudata *cpudata = policy->driver_data; | |
764 | ||
765 | perf = READ_ONCE(cpudata->highest_perf); | |
766 | ||
3ec32b6d | 767 | return sysfs_emit(buf, "%u\n", perf); |
3ad7fde1 HR |
768 | } |
769 | ||
ffa5096a PY |
770 | static ssize_t show_energy_performance_available_preferences( |
771 | struct cpufreq_policy *policy, char *buf) | |
772 | { | |
773 | int i = 0; | |
774 | int offset = 0; | |
775 | ||
776 | while (energy_perf_strings[i] != NULL) | |
777 | offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); | |
778 | ||
779 | sysfs_emit_at(buf, offset, "\n"); | |
780 | ||
781 | return offset; | |
782 | } | |
783 | ||
784 | static ssize_t store_energy_performance_preference( | |
785 | struct cpufreq_policy *policy, const char *buf, size_t count) | |
786 | { | |
787 | struct amd_cpudata *cpudata = policy->driver_data; | |
788 | char str_preference[21]; | |
789 | ssize_t ret; | |
790 | ||
791 | ret = sscanf(buf, "%20s", str_preference); | |
792 | if (ret != 1) | |
793 | return -EINVAL; | |
794 | ||
795 | ret = match_string(energy_perf_strings, -1, str_preference); | |
796 | if (ret < 0) | |
797 | return -EINVAL; | |
798 | ||
799 | mutex_lock(&amd_pstate_limits_lock); | |
800 | ret = amd_pstate_set_energy_pref_index(cpudata, ret); | |
801 | mutex_unlock(&amd_pstate_limits_lock); | |
802 | ||
803 | return ret ?: count; | |
804 | } | |
805 | ||
806 | static ssize_t show_energy_performance_preference( | |
807 | struct cpufreq_policy *policy, char *buf) | |
808 | { | |
809 | struct amd_cpudata *cpudata = policy->driver_data; | |
810 | int preference; | |
811 | ||
812 | preference = amd_pstate_get_energy_pref_index(cpudata); | |
813 | if (preference < 0) | |
814 | return preference; | |
815 | ||
816 | return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); | |
817 | } | |
818 | ||
abd61c08 PY |
819 | static ssize_t amd_pstate_show_status(char *buf) |
820 | { | |
821 | if (!current_pstate_driver) | |
822 | return sysfs_emit(buf, "disable\n"); | |
823 | ||
824 | return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]); | |
825 | } | |
826 | ||
827 | static void amd_pstate_driver_cleanup(void) | |
828 | { | |
829 | current_pstate_driver = NULL; | |
830 | } | |
831 | ||
832 | static int amd_pstate_update_status(const char *buf, size_t size) | |
833 | { | |
dd329e1e | 834 | int ret = 0; |
abd61c08 PY |
835 | int mode_idx; |
836 | ||
837 | if (size > 7 || size < 6) | |
838 | return -EINVAL; | |
839 | mode_idx = get_mode_idx_from_str(buf, size); | |
840 | ||
841 | switch(mode_idx) { | |
842 | case AMD_PSTATE_DISABLE: | |
843 | if (!current_pstate_driver) | |
844 | return -EINVAL; | |
845 | if (cppc_state == AMD_PSTATE_ACTIVE) | |
846 | return -EBUSY; | |
dd329e1e | 847 | cpufreq_unregister_driver(current_pstate_driver); |
abd61c08 PY |
848 | amd_pstate_driver_cleanup(); |
849 | break; | |
850 | case AMD_PSTATE_PASSIVE: | |
851 | if (current_pstate_driver) { | |
852 | if (current_pstate_driver == &amd_pstate_driver) | |
853 | return 0; | |
854 | cpufreq_unregister_driver(current_pstate_driver); | |
855 | cppc_state = AMD_PSTATE_PASSIVE; | |
856 | current_pstate_driver = &amd_pstate_driver; | |
857 | } | |
858 | ||
859 | ret = cpufreq_register_driver(current_pstate_driver); | |
860 | break; | |
861 | case AMD_PSTATE_ACTIVE: | |
862 | if (current_pstate_driver) { | |
863 | if (current_pstate_driver == &amd_pstate_epp_driver) | |
864 | return 0; | |
865 | cpufreq_unregister_driver(current_pstate_driver); | |
866 | current_pstate_driver = &amd_pstate_epp_driver; | |
867 | cppc_state = AMD_PSTATE_ACTIVE; | |
868 | } | |
869 | ||
870 | ret = cpufreq_register_driver(current_pstate_driver); | |
871 | break; | |
872 | default: | |
873 | ret = -EINVAL; | |
874 | break; | |
875 | } | |
876 | ||
877 | return ret; | |
878 | } | |
879 | ||
880 | static ssize_t show_status(struct kobject *kobj, | |
881 | struct kobj_attribute *attr, char *buf) | |
882 | { | |
883 | ssize_t ret; | |
884 | ||
885 | mutex_lock(&amd_pstate_driver_lock); | |
886 | ret = amd_pstate_show_status(buf); | |
887 | mutex_unlock(&amd_pstate_driver_lock); | |
888 | ||
889 | return ret; | |
890 | } | |
891 | ||
892 | static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, | |
893 | const char *buf, size_t count) | |
894 | { | |
895 | char *p = memchr(buf, '\n', count); | |
896 | int ret; | |
897 | ||
898 | mutex_lock(&amd_pstate_driver_lock); | |
899 | ret = amd_pstate_update_status(buf, p ? p - buf : count); | |
900 | mutex_unlock(&amd_pstate_driver_lock); | |
901 | ||
902 | return ret < 0 ? ret : count; | |
903 | } | |
904 | ||
ec4e3326 HR |
905 | cpufreq_freq_attr_ro(amd_pstate_max_freq); |
906 | cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); | |
907 | ||
3ad7fde1 | 908 | cpufreq_freq_attr_ro(amd_pstate_highest_perf); |
ffa5096a PY |
909 | cpufreq_freq_attr_rw(energy_performance_preference); |
910 | cpufreq_freq_attr_ro(energy_performance_available_preferences); | |
abd61c08 | 911 | define_one_global_rw(status); |
3ad7fde1 | 912 | |
ec4e3326 HR |
913 | static struct freq_attr *amd_pstate_attr[] = { |
914 | &amd_pstate_max_freq, | |
915 | &amd_pstate_lowest_nonlinear_freq, | |
3ad7fde1 | 916 | &amd_pstate_highest_perf, |
ec4e3326 HR |
917 | NULL, |
918 | }; | |
919 | ||
ffa5096a PY |
920 | static struct freq_attr *amd_pstate_epp_attr[] = { |
921 | &amd_pstate_max_freq, | |
922 | &amd_pstate_lowest_nonlinear_freq, | |
923 | &amd_pstate_highest_perf, | |
924 | &energy_performance_preference, | |
925 | &energy_performance_available_preferences, | |
926 | NULL, | |
927 | }; | |
928 | ||
abd61c08 PY |
929 | static struct attribute *pstate_global_attributes[] = { |
930 | &status.attr, | |
931 | NULL | |
932 | }; | |
933 | ||
934 | static const struct attribute_group amd_pstate_global_attr_group = { | |
935 | .attrs = pstate_global_attributes, | |
936 | }; | |
937 | ||
ffa5096a PY |
938 | static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) |
939 | { | |
940 | int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret; | |
941 | struct amd_cpudata *cpudata; | |
942 | struct device *dev; | |
ffa5096a PY |
943 | u64 value; |
944 | ||
945 | /* | |
946 | * Resetting PERF_CTL_MSR will put the CPU in P0 frequency, | |
947 | * which is ideal for initialization process. | |
948 | */ | |
949 | amd_perf_ctl_reset(policy->cpu); | |
950 | dev = get_cpu_device(policy->cpu); | |
951 | if (!dev) | |
7cca9a98 | 952 | return -ENODEV; |
ffa5096a PY |
953 | |
954 | cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL); | |
955 | if (!cpudata) | |
956 | return -ENOMEM; | |
957 | ||
958 | cpudata->cpu = policy->cpu; | |
959 | cpudata->epp_policy = 0; | |
960 | ||
7cca9a98 AB |
961 | ret = amd_pstate_init_perf(cpudata); |
962 | if (ret) | |
ffa5096a PY |
963 | goto free_cpudata1; |
964 | ||
965 | min_freq = amd_get_min_freq(cpudata); | |
966 | max_freq = amd_get_max_freq(cpudata); | |
967 | nominal_freq = amd_get_nominal_freq(cpudata); | |
968 | lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata); | |
969 | if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) { | |
970 | dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n", | |
971 | min_freq, max_freq); | |
972 | ret = -EINVAL; | |
973 | goto free_cpudata1; | |
974 | } | |
975 | ||
976 | policy->cpuinfo.min_freq = min_freq; | |
977 | policy->cpuinfo.max_freq = max_freq; | |
978 | /* It will be updated by governor */ | |
979 | policy->cur = policy->cpuinfo.min_freq; | |
980 | ||
981 | /* Initial processor data capability frequencies */ | |
982 | cpudata->max_freq = max_freq; | |
983 | cpudata->min_freq = min_freq; | |
984 | cpudata->nominal_freq = nominal_freq; | |
985 | cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq; | |
986 | ||
987 | policy->driver_data = cpudata; | |
988 | ||
989 | cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0); | |
990 | ||
991 | policy->min = policy->cpuinfo.min_freq; | |
992 | policy->max = policy->cpuinfo.max_freq; | |
993 | ||
994 | /* | |
995 | * Set the policy to powersave to provide a valid fallback value in case | |
996 | * the default cpufreq governor is neither powersave nor performance. | |
997 | */ | |
998 | policy->policy = CPUFREQ_POLICY_POWERSAVE; | |
999 | ||
1000 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1001 | policy->fast_switch_possible = true; | |
1002 | ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value); | |
1003 | if (ret) | |
1004 | return ret; | |
1005 | WRITE_ONCE(cpudata->cppc_req_cached, value); | |
1006 | ||
1007 | ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value); | |
1008 | if (ret) | |
1009 | return ret; | |
1010 | WRITE_ONCE(cpudata->cppc_cap1_cached, value); | |
1011 | } | |
1012 | amd_pstate_boost_init(cpudata); | |
1013 | ||
1014 | return 0; | |
1015 | ||
1016 | free_cpudata1: | |
1017 | kfree(cpudata); | |
1018 | return ret; | |
1019 | } | |
1020 | ||
1021 | static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) | |
1022 | { | |
1023 | pr_debug("CPU %d exiting\n", policy->cpu); | |
1024 | policy->fast_switch_possible = false; | |
1025 | return 0; | |
1026 | } | |
1027 | ||
1028 | static void amd_pstate_epp_init(unsigned int cpu) | |
1029 | { | |
1030 | struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); | |
1031 | struct amd_cpudata *cpudata = policy->driver_data; | |
1032 | u32 max_perf, min_perf; | |
1033 | u64 value; | |
1034 | s16 epp; | |
1035 | ||
1036 | max_perf = READ_ONCE(cpudata->highest_perf); | |
1037 | min_perf = READ_ONCE(cpudata->lowest_perf); | |
1038 | ||
1039 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1040 | ||
1041 | if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) | |
1042 | min_perf = max_perf; | |
1043 | ||
1044 | /* Initial min/max values for CPPC Performance Controls Register */ | |
1045 | value &= ~AMD_CPPC_MIN_PERF(~0L); | |
1046 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
1047 | ||
1048 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
1049 | value |= AMD_CPPC_MAX_PERF(max_perf); | |
1050 | ||
1051 | /* CPPC EPP feature require to set zero to the desire perf bit */ | |
1052 | value &= ~AMD_CPPC_DES_PERF(~0L); | |
1053 | value |= AMD_CPPC_DES_PERF(0); | |
1054 | ||
1055 | if (cpudata->epp_policy == cpudata->policy) | |
1056 | goto skip_epp; | |
1057 | ||
1058 | cpudata->epp_policy = cpudata->policy; | |
1059 | ||
6e9d1212 WK |
1060 | /* Get BIOS pre-defined epp value */ |
1061 | epp = amd_pstate_get_epp(cpudata, value); | |
1062 | if (epp < 0) { | |
1063 | /** | |
1064 | * This return value can only be negative for shared_memory | |
1065 | * systems where EPP register read/write not supported. | |
1066 | */ | |
1067 | goto skip_epp; | |
ffa5096a | 1068 | } |
6e9d1212 WK |
1069 | |
1070 | if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) | |
1071 | epp = 0; | |
1072 | ||
ffa5096a PY |
1073 | /* Set initial EPP value */ |
1074 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1075 | value &= ~GENMASK_ULL(31, 24); | |
1076 | value |= (u64)epp << 24; | |
1077 | } | |
1078 | ||
6e9d1212 | 1079 | WRITE_ONCE(cpudata->cppc_req_cached, value); |
7cca9a98 | 1080 | amd_pstate_set_epp(cpudata, epp); |
ffa5096a | 1081 | skip_epp: |
ffa5096a PY |
1082 | cpufreq_cpu_put(policy); |
1083 | } | |
1084 | ||
1085 | static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy) | |
1086 | { | |
1087 | struct amd_cpudata *cpudata = policy->driver_data; | |
1088 | ||
1089 | if (!policy->cpuinfo.max_freq) | |
1090 | return -ENODEV; | |
1091 | ||
1092 | pr_debug("set_policy: cpuinfo.max %u policy->max %u\n", | |
1093 | policy->cpuinfo.max_freq, policy->max); | |
1094 | ||
1095 | cpudata->policy = policy->policy; | |
1096 | ||
1097 | amd_pstate_epp_init(policy->cpu); | |
1098 | ||
1099 | return 0; | |
1100 | } | |
1101 | ||
d4da12f8 PY |
1102 | static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata) |
1103 | { | |
1104 | struct cppc_perf_ctrls perf_ctrls; | |
1105 | u64 value, max_perf; | |
1106 | int ret; | |
1107 | ||
1108 | ret = amd_pstate_enable(true); | |
1109 | if (ret) | |
1110 | pr_err("failed to enable amd pstate during resume, return %d\n", ret); | |
1111 | ||
1112 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1113 | max_perf = READ_ONCE(cpudata->highest_perf); | |
1114 | ||
1115 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1116 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
1117 | } else { | |
1118 | perf_ctrls.max_perf = max_perf; | |
1119 | perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached); | |
1120 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
1121 | } | |
1122 | } | |
1123 | ||
1124 | static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy) | |
1125 | { | |
1126 | struct amd_cpudata *cpudata = policy->driver_data; | |
1127 | ||
1128 | pr_debug("AMD CPU Core %d going online\n", cpudata->cpu); | |
1129 | ||
1130 | if (cppc_state == AMD_PSTATE_ACTIVE) { | |
1131 | amd_pstate_epp_reenable(cpudata); | |
1132 | cpudata->suspended = false; | |
1133 | } | |
1134 | ||
1135 | return 0; | |
1136 | } | |
1137 | ||
1138 | static void amd_pstate_epp_offline(struct cpufreq_policy *policy) | |
1139 | { | |
1140 | struct amd_cpudata *cpudata = policy->driver_data; | |
1141 | struct cppc_perf_ctrls perf_ctrls; | |
1142 | int min_perf; | |
1143 | u64 value; | |
1144 | ||
1145 | min_perf = READ_ONCE(cpudata->lowest_perf); | |
1146 | value = READ_ONCE(cpudata->cppc_req_cached); | |
1147 | ||
1148 | mutex_lock(&amd_pstate_limits_lock); | |
1149 | if (boot_cpu_has(X86_FEATURE_CPPC)) { | |
1150 | cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN; | |
1151 | ||
1152 | /* Set max perf same as min perf */ | |
1153 | value &= ~AMD_CPPC_MAX_PERF(~0L); | |
1154 | value |= AMD_CPPC_MAX_PERF(min_perf); | |
1155 | value &= ~AMD_CPPC_MIN_PERF(~0L); | |
1156 | value |= AMD_CPPC_MIN_PERF(min_perf); | |
1157 | wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value); | |
1158 | } else { | |
1159 | perf_ctrls.desired_perf = 0; | |
1160 | perf_ctrls.max_perf = min_perf; | |
1161 | perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE); | |
1162 | cppc_set_perf(cpudata->cpu, &perf_ctrls); | |
1163 | } | |
1164 | mutex_unlock(&amd_pstate_limits_lock); | |
1165 | } | |
1166 | ||
1167 | static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy) | |
1168 | { | |
1169 | struct amd_cpudata *cpudata = policy->driver_data; | |
1170 | ||
1171 | pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu); | |
1172 | ||
1173 | if (cpudata->suspended) | |
1174 | return 0; | |
1175 | ||
1176 | if (cppc_state == AMD_PSTATE_ACTIVE) | |
1177 | amd_pstate_epp_offline(policy); | |
1178 | ||
1179 | return 0; | |
1180 | } | |
1181 | ||
ffa5096a PY |
1182 | static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy) |
1183 | { | |
1184 | cpufreq_verify_within_cpu_limits(policy); | |
1185 | pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min); | |
1186 | return 0; | |
1187 | } | |
1188 | ||
50ddd2f7 PY |
1189 | static int amd_pstate_epp_suspend(struct cpufreq_policy *policy) |
1190 | { | |
1191 | struct amd_cpudata *cpudata = policy->driver_data; | |
1192 | int ret; | |
1193 | ||
1194 | /* avoid suspending when EPP is not enabled */ | |
1195 | if (cppc_state != AMD_PSTATE_ACTIVE) | |
1196 | return 0; | |
1197 | ||
1198 | /* set this flag to avoid setting core offline*/ | |
1199 | cpudata->suspended = true; | |
1200 | ||
1201 | /* disable CPPC in lowlevel firmware */ | |
1202 | ret = amd_pstate_enable(false); | |
1203 | if (ret) | |
1204 | pr_err("failed to suspend, return %d\n", ret); | |
1205 | ||
1206 | return 0; | |
1207 | } | |
1208 | ||
1209 | static int amd_pstate_epp_resume(struct cpufreq_policy *policy) | |
1210 | { | |
1211 | struct amd_cpudata *cpudata = policy->driver_data; | |
1212 | ||
1213 | if (cpudata->suspended) { | |
1214 | mutex_lock(&amd_pstate_limits_lock); | |
1215 | ||
1216 | /* enable amd pstate from suspend state*/ | |
1217 | amd_pstate_epp_reenable(cpudata); | |
1218 | ||
1219 | mutex_unlock(&amd_pstate_limits_lock); | |
1220 | ||
1221 | cpudata->suspended = false; | |
1222 | } | |
1223 | ||
1224 | return 0; | |
1225 | } | |
1226 | ||
ec437d71 HR |
1227 | static struct cpufreq_driver amd_pstate_driver = { |
1228 | .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS, | |
1229 | .verify = amd_pstate_verify, | |
1230 | .target = amd_pstate_target, | |
1231 | .init = amd_pstate_cpu_init, | |
1232 | .exit = amd_pstate_cpu_exit, | |
b376471f JS |
1233 | .suspend = amd_pstate_cpu_suspend, |
1234 | .resume = amd_pstate_cpu_resume, | |
41271016 | 1235 | .set_boost = amd_pstate_set_boost, |
ec437d71 | 1236 | .name = "amd-pstate", |
d8bee41d | 1237 | .attr = amd_pstate_attr, |
ec437d71 HR |
1238 | }; |
1239 | ||
ffa5096a PY |
1240 | static struct cpufreq_driver amd_pstate_epp_driver = { |
1241 | .flags = CPUFREQ_CONST_LOOPS, | |
1242 | .verify = amd_pstate_epp_verify_policy, | |
1243 | .setpolicy = amd_pstate_epp_set_policy, | |
1244 | .init = amd_pstate_epp_cpu_init, | |
1245 | .exit = amd_pstate_epp_cpu_exit, | |
d4da12f8 PY |
1246 | .offline = amd_pstate_epp_cpu_offline, |
1247 | .online = amd_pstate_epp_cpu_online, | |
50ddd2f7 PY |
1248 | .suspend = amd_pstate_epp_suspend, |
1249 | .resume = amd_pstate_epp_resume, | |
ffa5096a PY |
1250 | .name = "amd_pstate_epp", |
1251 | .attr = amd_pstate_epp_attr, | |
1252 | }; | |
1253 | ||
ec437d71 HR |
1254 | static int __init amd_pstate_init(void) |
1255 | { | |
1256 | int ret; | |
1257 | ||
1258 | if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) | |
1259 | return -ENODEV; | |
202e683d PY |
1260 | /* |
1261 | * by default the pstate driver is disabled to load | |
1262 | * enable the amd_pstate passive mode driver explicitly | |
36c5014e | 1263 | * with amd_pstate=passive or other modes in kernel command line |
202e683d | 1264 | */ |
36c5014e WK |
1265 | if (cppc_state == AMD_PSTATE_DISABLE) { |
1266 | pr_debug("driver load is disabled, boot with specific mode to enable this\n"); | |
202e683d PY |
1267 | return -ENODEV; |
1268 | } | |
ec437d71 HR |
1269 | |
1270 | if (!acpi_cpc_valid()) { | |
a2a9d185 | 1271 | pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n"); |
ec437d71 HR |
1272 | return -ENODEV; |
1273 | } | |
1274 | ||
1275 | /* don't keep reloading if cpufreq_driver exists */ | |
1276 | if (cpufreq_get_current_driver()) | |
1277 | return -EEXIST; | |
1278 | ||
1279 | /* capability check */ | |
e059c184 HR |
1280 | if (boot_cpu_has(X86_FEATURE_CPPC)) { |
1281 | pr_debug("AMD CPPC MSR based functionality is supported\n"); | |
ffa5096a PY |
1282 | if (cppc_state == AMD_PSTATE_PASSIVE) |
1283 | current_pstate_driver->adjust_perf = amd_pstate_adjust_perf; | |
202e683d PY |
1284 | } else { |
1285 | pr_debug("AMD CPPC shared memory based functionality is supported\n"); | |
e059c184 HR |
1286 | static_call_update(amd_pstate_enable, cppc_enable); |
1287 | static_call_update(amd_pstate_init_perf, cppc_init_perf); | |
1288 | static_call_update(amd_pstate_update_perf, cppc_update_perf); | |
ec437d71 HR |
1289 | } |
1290 | ||
1291 | /* enable amd pstate feature */ | |
1292 | ret = amd_pstate_enable(true); | |
1293 | if (ret) { | |
ffa5096a | 1294 | pr_err("failed to enable with return %d\n", ret); |
ec437d71 HR |
1295 | return ret; |
1296 | } | |
1297 | ||
ffa5096a | 1298 | ret = cpufreq_register_driver(current_pstate_driver); |
ec437d71 | 1299 | if (ret) |
ffa5096a | 1300 | pr_err("failed to register with return %d\n", ret); |
ec437d71 | 1301 | |
abd61c08 PY |
1302 | amd_pstate_kobj = kobject_create_and_add("amd_pstate", &cpu_subsys.dev_root->kobj); |
1303 | if (!amd_pstate_kobj) { | |
1304 | ret = -EINVAL; | |
1305 | pr_err("global sysfs registration failed.\n"); | |
1306 | goto kobject_free; | |
1307 | } | |
1308 | ||
1309 | ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group); | |
1310 | if (ret) { | |
1311 | pr_err("sysfs attribute export failed with error %d.\n", ret); | |
1312 | goto global_attr_free; | |
1313 | } | |
1314 | ||
1315 | return ret; | |
1316 | ||
1317 | global_attr_free: | |
1318 | kobject_put(amd_pstate_kobj); | |
1319 | kobject_free: | |
1320 | cpufreq_unregister_driver(current_pstate_driver); | |
ec437d71 HR |
1321 | return ret; |
1322 | } | |
456ca88d | 1323 | device_initcall(amd_pstate_init); |
ec437d71 | 1324 | |
202e683d PY |
1325 | static int __init amd_pstate_param(char *str) |
1326 | { | |
36c5014e WK |
1327 | size_t size; |
1328 | int mode_idx; | |
1329 | ||
202e683d PY |
1330 | if (!str) |
1331 | return -EINVAL; | |
1332 | ||
36c5014e WK |
1333 | size = strlen(str); |
1334 | mode_idx = get_mode_idx_from_str(str, size); | |
202e683d | 1335 | |
36c5014e WK |
1336 | if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) { |
1337 | cppc_state = mode_idx; | |
1338 | if (cppc_state == AMD_PSTATE_DISABLE) | |
1339 | pr_info("driver is explicitly disabled\n"); | |
1340 | ||
ffa5096a PY |
1341 | if (cppc_state == AMD_PSTATE_ACTIVE) |
1342 | current_pstate_driver = &amd_pstate_epp_driver; | |
1343 | ||
1344 | if (cppc_state == AMD_PSTATE_PASSIVE) | |
1345 | current_pstate_driver = &amd_pstate_driver; | |
1346 | ||
36c5014e WK |
1347 | return 0; |
1348 | } | |
1349 | ||
1350 | return -EINVAL; | |
202e683d PY |
1351 | } |
1352 | early_param("amd_pstate", amd_pstate_param); | |
1353 | ||
ec437d71 HR |
1354 | MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>"); |
1355 | MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver"); | |
1356 | MODULE_LICENSE("GPL"); |