projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'sched_ext-for-6.12-rc1-fixes-1' of git://git.kernel.org/pub/scm/linux...
[linux-block.git] / drivers / cpufreq / amd-pstate.c
CommitLineData
ec437d71
HR		 1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * amd-pstate.c - AMD Processor P-state Frequency Driver
4 *
5 * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
6 *
7 * Author: Huang Rui <ray.huang@amd.com>
8 *
9 * AMD P-State introduces a new CPU performance scaling design for AMD
10 * processors using the ACPI Collaborative Performance and Power Control (CPPC)
11 * feature which works with the AMD SMU firmware providing a finer grained
12 * frequency control range. It is to replace the legacy ACPI P-States control,
13 * allows a flexible, low-latency interface for the Linux kernel to directly
14 * communicate the performance hints to hardware.
15 *
16 * AMD P-State is supported on recent AMD Zen base CPU series include some of
17 * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
18 * P-State supported system. And there are two types of hardware implementations
19 * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
20 * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
21 */
22
23#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24
25#include <linux/kernel.h>
26#include <linux/module.h>
27#include <linux/init.h>
28#include <linux/smp.h>
29#include <linux/sched.h>
30#include <linux/cpufreq.h>
31#include <linux/compiler.h>
32#include <linux/dmi.h>
33#include <linux/slab.h>
34#include <linux/acpi.h>
35#include <linux/io.h>
36#include <linux/delay.h>
37#include <linux/uaccess.h>
38#include <linux/static_call.h>
f3a05239 39#include <linux/topology.h>
ec437d71
HR		 40
41#include <acpi/processor.h>
42#include <acpi/cppc_acpi.h>
43
44#include <asm/msr.h>
45#include <asm/processor.h>
46#include <asm/cpufeature.h>
47#include <asm/cpu_device_id.h>
779b8a14
AB		 48
49#include "amd-pstate.h"
60e10f89 50#include "amd-pstate-trace.h"
ec437d71 51
ca08e46d
PY		 52#define AMD_PSTATE_TRANSITION_LATENCY 20000
53#define AMD_PSTATE_TRANSITION_DELAY 1000
c00d476c 54#define AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY 600
ec437d71 55
779b8a14
AB		 56#define AMD_CPPC_EPP_PERFORMANCE 0x00
57#define AMD_CPPC_EPP_BALANCE_PERFORMANCE 0x80
58#define AMD_CPPC_EPP_BALANCE_POWERSAVE 0xBF
59#define AMD_CPPC_EPP_POWERSAVE 0xFF
60
779b8a14
AB		 61static const char * const amd_pstate_mode_string[] = {
62 [AMD_PSTATE_UNDEFINED] = "undefined",
63 [AMD_PSTATE_DISABLE] = "disable",
64 [AMD_PSTATE_PASSIVE] = "passive",
65 [AMD_PSTATE_ACTIVE] = "active",
66 [AMD_PSTATE_GUIDED] = "guided",
67 NULL,
68};
69
8d916815
ML		 70const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode)
71{
72 if (mode < 0 || mode >= AMD_PSTATE_MAX)
73 return NULL;
74 return amd_pstate_mode_string[mode];
75}
76EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string);
77
779b8a14
AB		 78struct quirk_entry {
79 u32 nominal_freq;
80 u32 lowest_freq;
81};
82
ffa5096a 83static struct cpufreq_driver *current_pstate_driver;
ec437d71 84static struct cpufreq_driver amd_pstate_driver;
ffa5096a 85static struct cpufreq_driver amd_pstate_epp_driver;
c88ad30e 86static int cppc_state = AMD_PSTATE_UNDEFINED;
217e6778 87static bool cppc_enabled;
f3a05239 88static bool amd_pstate_prefcore = true;
eb8b6c36 89static struct quirk_entry *quirks;
36c5014e 90
ffa5096a
PY		 91/*
92 * AMD Energy Preference Performance (EPP)
93 * The EPP is used in the CCLK DPM controller to drive
94 * the frequency that a core is going to operate during
95 * short periods of activity. EPP values will be utilized for
96 * different OS profiles (balanced, performance, power savings)
97 * display strings corresponding to EPP index in the
98 * energy_perf_strings[]
99 * index String
100 *-------------------------------------
101 * 0 default
102 * 1 performance
103 * 2 balance_performance
104 * 3 balance_power
105 * 4 power
106 */
107enum energy_perf_value_index {
108 EPP_INDEX_DEFAULT = 0,
109 EPP_INDEX_PERFORMANCE,
110 EPP_INDEX_BALANCE_PERFORMANCE,
111 EPP_INDEX_BALANCE_POWERSAVE,
112 EPP_INDEX_POWERSAVE,
113};
114
115static const char * const energy_perf_strings[] = {
116 [EPP_INDEX_DEFAULT] = "default",
117 [EPP_INDEX_PERFORMANCE] = "performance",
118 [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
119 [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
120 [EPP_INDEX_POWERSAVE] = "power",
121 NULL
122};
123
124static unsigned int epp_values[] = {
125 [EPP_INDEX_DEFAULT] = 0,
126 [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
127 [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
128 [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
129 [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
130 };
131
3ca7bc81
WK		 132typedef int (*cppc_mode_transition_fn)(int);
133
eb8b6c36
PY		 134static struct quirk_entry quirk_amd_7k62 = {
135 .nominal_freq = 2600,
136 .lowest_freq = 550,
137};
138
139static int __init dmi_matched_7k62_bios_bug(const struct dmi_system_id *dmi)
140{
141 /**
142 * match the broken bios for family 17h processor support CPPC V2
143 * broken BIOS lack of nominal_freq and lowest_freq capabilities
144 * definition in ACPI tables
145 */
c9fdaba8 146 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) {
eb8b6c36
PY		 147 quirks = dmi->driver_data;
148 pr_info("Overriding nominal and lowest frequencies for %s\n", dmi->ident);
149 return 1;
150 }
151
152 return 0;
153}
154
155static const struct dmi_system_id amd_pstate_quirks_table[] __initconst = {
156 {
157 .callback = dmi_matched_7k62_bios_bug,
158 .ident = "AMD EPYC 7K62",
159 .matches = {
160 DMI_MATCH(DMI_BIOS_VERSION, "5.14"),
161 DMI_MATCH(DMI_BIOS_RELEASE, "12/12/2019"),
162 },
163 .driver_data = &quirk_amd_7k62,
164 },
165 {}
166};
167MODULE_DEVICE_TABLE(dmi, amd_pstate_quirks_table);
168
36c5014e
WK		 169static inline int get_mode_idx_from_str(const char *str, size_t size)
170{
171 int i;
172
173 for (i=0; i < AMD_PSTATE_MAX; i++) {
174 if (!strncmp(str, amd_pstate_mode_string[i], size))
175 return i;
176 }
177 return -EINVAL;
178}
ec437d71 179
ffa5096a
PY		 180static DEFINE_MUTEX(amd_pstate_limits_lock);
181static DEFINE_MUTEX(amd_pstate_driver_lock);
182
183static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)
184{
185 u64 epp;
186 int ret;
187
c9fdaba8 188 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
ffa5096a
PY		 189 if (!cppc_req_cached) {
190 epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
191 &cppc_req_cached);
192 if (epp)
193 return epp;
194 }
195 epp = (cppc_req_cached >> 24) & 0xFF;
196 } else {
197 ret = cppc_get_epp_perf(cpudata->cpu, &epp);
198 if (ret < 0) {
199 pr_debug("Could not retrieve energy perf value (%d)\n", ret);
200 return -EIO;
201 }
202 }
203
204 return (s16)(epp & 0xff);
205}
206
207static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)
208{
209 s16 epp;
210 int index = -EINVAL;
211
212 epp = amd_pstate_get_epp(cpudata, 0);
213 if (epp < 0)
214 return epp;
215
216 switch (epp) {
217 case AMD_CPPC_EPP_PERFORMANCE:
218 index = EPP_INDEX_PERFORMANCE;
219 break;
220 case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
221 index = EPP_INDEX_BALANCE_PERFORMANCE;
222 break;
223 case AMD_CPPC_EPP_BALANCE_POWERSAVE:
224 index = EPP_INDEX_BALANCE_POWERSAVE;
225 break;
226 case AMD_CPPC_EPP_POWERSAVE:
227 index = EPP_INDEX_POWERSAVE;
228 break;
229 default:
230 break;
231 }
232
233 return index;
234}
235
738d7d03
DU		 236static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
237 u32 des_perf, u32 max_perf, bool fast_switch)
238{
239 if (fast_switch)
240 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
241 else
242 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
243 READ_ONCE(cpudata->cppc_req_cached));
244}
245
246DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
247
248static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
249 u32 min_perf, u32 des_perf,
250 u32 max_perf, bool fast_switch)
251{
252 static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
253 max_perf, fast_switch);
254}
255
ffa5096a
PY		 256static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)
257{
258 int ret;
259 struct cppc_perf_ctrls perf_ctrls;
260
c9fdaba8 261 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
ffa5096a
PY		 262 u64 value = READ_ONCE(cpudata->cppc_req_cached);
263
264 value &= ~GENMASK_ULL(31, 24);
265 value |= (u64)epp << 24;
266 WRITE_ONCE(cpudata->cppc_req_cached, value);
267
268 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
269 if (!ret)
270 cpudata->epp_cached = epp;
271 } else {
738d7d03
DU		 272 amd_pstate_update_perf(cpudata, cpudata->min_limit_perf, 0U,
273 cpudata->max_limit_perf, false);
274
ffa5096a
PY		 275 perf_ctrls.energy_perf = epp;
276 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
277 if (ret) {
278 pr_debug("failed to set energy perf value (%d)\n", ret);
279 return ret;
280 }
281 cpudata->epp_cached = epp;
282 }
283
284 return ret;
285}
286
287static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,
288 int pref_index)
289{
290 int epp = -EINVAL;
291 int ret;
292
fc6e0837
ML		 293 if (!pref_index)
294 epp = cpudata->epp_default;
ffa5096a
PY		 295
296 if (epp == -EINVAL)
297 epp = epp_values[pref_index];
298
299 if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
300 pr_debug("EPP cannot be set under performance policy\n");
301 return -EBUSY;
302 }
303
304 ret = amd_pstate_set_epp(cpudata, epp);
305
306 return ret;
307}
308
e059c184 309static inline int pstate_enable(bool enable)
ec437d71 310{
217e6778
WK		 311 int ret, cpu;
312 unsigned long logical_proc_id_mask = 0;
313
314 if (enable == cppc_enabled)
315 return 0;
316
317 for_each_present_cpu(cpu) {
0d8584d2 318 unsigned long logical_id = topology_logical_package_id(cpu);
217e6778
WK		 319
320 if (test_bit(logical_id, &logical_proc_id_mask))
321 continue;
322
323 set_bit(logical_id, &logical_proc_id_mask);
324
325 ret = wrmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_ENABLE,
326 enable);
327 if (ret)
328 return ret;
329 }
330
331 cppc_enabled = enable;
332 return 0;
ec437d71
HR		 333}
334
e059c184
HR		 335static int cppc_enable(bool enable)
336{
337 int cpu, ret = 0;
ffa5096a 338 struct cppc_perf_ctrls perf_ctrls;
e059c184 339
217e6778
WK		 340 if (enable == cppc_enabled)
341 return 0;
342
e059c184
HR		 343 for_each_present_cpu(cpu) {
344 ret = cppc_set_enable(cpu, enable);
345 if (ret)
346 return ret;
ffa5096a
PY		 347
348 /* Enable autonomous mode for EPP */
349 if (cppc_state == AMD_PSTATE_ACTIVE) {
350 /* Set desired perf as zero to allow EPP firmware control */
351 perf_ctrls.desired_perf = 0;
352 ret = cppc_set_perf(cpu, &perf_ctrls);
353 if (ret)
354 return ret;
355 }
e059c184
HR		 356 }
357
217e6778 358 cppc_enabled = enable;
e059c184
HR		 359 return ret;
360}
361
362DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
363
364static inline int amd_pstate_enable(bool enable)
365{
366 return static_call(amd_pstate_enable)(enable);
367}
368
369static int pstate_init_perf(struct amd_cpudata *cpudata)
ec437d71
HR		 370{
371 u64 cap1;
372
373 int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
374 &cap1);
375 if (ret)
376 return ret;
377
ad4caad5
ML		 378 WRITE_ONCE(cpudata->highest_perf, AMD_CPPC_HIGHEST_PERF(cap1));
379 WRITE_ONCE(cpudata->max_limit_perf, AMD_CPPC_HIGHEST_PERF(cap1));
ec437d71
HR		 380 WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
381 WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
382 WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
e571a5e2 383 WRITE_ONCE(cpudata->prefcore_ranking, AMD_CPPC_HIGHEST_PERF(cap1));
febab20c 384 WRITE_ONCE(cpudata->min_limit_perf, AMD_CPPC_LOWEST_PERF(cap1));
ec437d71
HR		 385 return 0;
386}
387
e059c184
HR		 388static int cppc_init_perf(struct amd_cpudata *cpudata)
389{
390 struct cppc_perf_caps cppc_perf;
391
392 int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
393 if (ret)
394 return ret;
395
ad4caad5
ML		 396 WRITE_ONCE(cpudata->highest_perf, cppc_perf.highest_perf);
397 WRITE_ONCE(cpudata->max_limit_perf, cppc_perf.highest_perf);
e059c184
HR		 398 WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
399 WRITE_ONCE(cpudata->lowest_nonlinear_perf,
400 cppc_perf.lowest_nonlinear_perf);
401 WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
e571a5e2 402 WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf);
febab20c 403 WRITE_ONCE(cpudata->min_limit_perf, cppc_perf.lowest_perf);
e059c184 404
2dd6d0eb
WK		 405 if (cppc_state == AMD_PSTATE_ACTIVE)
406 return 0;
407
408 ret = cppc_get_auto_sel_caps(cpudata->cpu, &cppc_perf);
409 if (ret) {
410 pr_warn("failed to get auto_sel, ret: %d\n", ret);
411 return 0;
412 }
413
414 ret = cppc_set_auto_sel(cpudata->cpu,
415 (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
416
417 if (ret)
418 pr_warn("failed to set auto_sel, ret: %d\n", ret);
419
420 return ret;
e059c184
HR		 421}
422
423DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
424
425static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
426{
427 return static_call(amd_pstate_init_perf)(cpudata);
428}
429
e059c184
HR		 430static void cppc_update_perf(struct amd_cpudata *cpudata,
431 u32 min_perf, u32 des_perf,
432 u32 max_perf, bool fast_switch)
433{
434 struct cppc_perf_ctrls perf_ctrls;
435
436 perf_ctrls.max_perf = max_perf;
437 perf_ctrls.min_perf = min_perf;
438 perf_ctrls.desired_perf = des_perf;
439
440 cppc_set_perf(cpudata->cpu, &perf_ctrls);
441}
442
23c296fb
JS		 443static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
444{
445 u64 aperf, mperf, tsc;
446 unsigned long flags;
447
448 local_irq_save(flags);
449 rdmsrl(MSR_IA32_APERF, aperf);
450 rdmsrl(MSR_IA32_MPERF, mperf);
451 tsc = rdtsc();
452
453 if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
454 local_irq_restore(flags);
455 return false;
456 }
457
458 local_irq_restore(flags);
459
460 cpudata->cur.aperf = aperf;
461 cpudata->cur.mperf = mperf;
462 cpudata->cur.tsc = tsc;
463 cpudata->cur.aperf -= cpudata->prev.aperf;
464 cpudata->cur.mperf -= cpudata->prev.mperf;
465 cpudata->cur.tsc -= cpudata->prev.tsc;
466
467 cpudata->prev.aperf = aperf;
468 cpudata->prev.mperf = mperf;
469 cpudata->prev.tsc = tsc;
470
471 cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
472
473 return true;
474}
475
ec437d71 476static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
2dd6d0eb 477 u32 des_perf, u32 max_perf, bool fast_switch, int gov_flags)
ec437d71 478{
e8f555da
ML		 479 unsigned long max_freq;
480 struct cpufreq_policy *policy = cpufreq_cpu_get(cpudata->cpu);
ec437d71 481 u64 prev = READ_ONCE(cpudata->cppc_req_cached);
89ac482d 482 u32 nominal_perf = READ_ONCE(cpudata->nominal_perf);
ec437d71
HR		 483 u64 value = prev;
484
febab20c
WK		 485 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
486 cpudata->max_limit_perf);
487 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
488 cpudata->max_limit_perf);
0e9a8638 489 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
2dd6d0eb 490
e8f555da
ML		 491 max_freq = READ_ONCE(cpudata->max_limit_freq);
492 policy->cur = div_u64(des_perf * max_freq, max_perf);
493
2dd6d0eb
WK		 494 if ((cppc_state == AMD_PSTATE_GUIDED) && (gov_flags & CPUFREQ_GOV_DYNAMIC_SWITCHING)) {
495 min_perf = des_perf;
496 des_perf = 0;
497 }
498
ec437d71
HR		 499 value &= ~AMD_CPPC_MIN_PERF(~0L);
500 value |= AMD_CPPC_MIN_PERF(min_perf);
501
502 value &= ~AMD_CPPC_DES_PERF(~0L);
503 value |= AMD_CPPC_DES_PERF(des_perf);
504
89ac482d
PY		 505 /* limit the max perf when core performance boost feature is disabled */
506 if (!cpudata->boost_supported)
507 max_perf = min_t(unsigned long, nominal_perf, max_perf);
508
ec437d71
HR		 509 value &= ~AMD_CPPC_MAX_PERF(~0L);
510 value |= AMD_CPPC_MAX_PERF(max_perf);
511
23c296fb
JS		 512 if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
513 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
514 cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
515 cpudata->cpu, (value != prev), fast_switch);
516 }
60e10f89 517
ec437d71 518 if (value == prev)
49243adc 519 goto cpufreq_policy_put;
ec437d71
HR		 520
521 WRITE_ONCE(cpudata->cppc_req_cached, value);
522
523 amd_pstate_update_perf(cpudata, min_perf, des_perf,
524 max_perf, fast_switch);
49243adc
DU		 525
526cpufreq_policy_put:
527 cpufreq_cpu_put(policy);
ec437d71
HR		 528}
529
530static int amd_pstate_verify(struct cpufreq_policy_data *policy)
531{
532 cpufreq_verify_within_cpu_limits(policy);
533
534 return 0;
535}
536
febab20c
WK		 537static int amd_pstate_update_min_max_limit(struct cpufreq_policy *policy)
538{
8164f743 539 u32 max_limit_perf, min_limit_perf, lowest_perf;
febab20c
WK		 540 struct amd_cpudata *cpudata = policy->driver_data;
541
542 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
543 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
544
8164f743
ML		 545 lowest_perf = READ_ONCE(cpudata->lowest_perf);
546 if (min_limit_perf < lowest_perf)
547 min_limit_perf = lowest_perf;
548
549 if (max_limit_perf < min_limit_perf)
550 max_limit_perf = min_limit_perf;
551
febab20c
WK		 552 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
553 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
554 WRITE_ONCE(cpudata->max_limit_freq, policy->max);
555 WRITE_ONCE(cpudata->min_limit_freq, policy->min);
556
557 return 0;
558}
559
4badf2eb
GS		 560static int amd_pstate_update_freq(struct cpufreq_policy *policy,
561 unsigned int target_freq, bool fast_switch)
ec437d71
HR		 562{
563 struct cpufreq_freqs freqs;
564 struct amd_cpudata *cpudata = policy->driver_data;
565 unsigned long max_perf, min_perf, des_perf, cap_perf;
566
567 if (!cpudata->max_freq)
568 return -ENODEV;
569
febab20c
WK		 570 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
571 amd_pstate_update_min_max_limit(policy);
572
ec437d71 573 cap_perf = READ_ONCE(cpudata->highest_perf);
b185c505 574 min_perf = READ_ONCE(cpudata->lowest_perf);
ec437d71
HR		 575 max_perf = cap_perf;
576
577 freqs.old = policy->cur;
578 freqs.new = target_freq;
579
580 des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
581 cpudata->max_freq);
582
4badf2eb
GS		 583 WARN_ON(fast_switch && !policy->fast_switch_enabled);
584 /*
585 * If fast_switch is desired, then there aren't any registered
586 * transition notifiers. See comment for
587 * cpufreq_enable_fast_switch().
588 */
589 if (!fast_switch)
590 cpufreq_freq_transition_begin(policy, &freqs);
591
ec437d71 592 amd_pstate_update(cpudata, min_perf, des_perf,
4badf2eb
GS		 593 max_perf, fast_switch, policy->governor->flags);
594
595 if (!fast_switch)
596 cpufreq_freq_transition_end(policy, &freqs, false);
ec437d71
HR		 597
598 return 0;
599}
600
4badf2eb
GS		 601static int amd_pstate_target(struct cpufreq_policy *policy,
602 unsigned int target_freq,
603 unsigned int relation)
604{
605 return amd_pstate_update_freq(policy, target_freq, false);
606}
607
608static unsigned int amd_pstate_fast_switch(struct cpufreq_policy *policy,
609 unsigned int target_freq)
610{
bb87be26
GS		 611 if (!amd_pstate_update_freq(policy, target_freq, true))
612 return target_freq;
613 return policy->cur;
4badf2eb
GS		 614}
615
1d215f03
HR		 616static void amd_pstate_adjust_perf(unsigned int cpu,
617 unsigned long _min_perf,
618 unsigned long target_perf,
619 unsigned long capacity)
620{
621 unsigned long max_perf, min_perf, des_perf,
e8f555da 622 cap_perf, lowest_nonlinear_perf;
1d215f03 623 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
5493f971
AB		 624 struct amd_cpudata *cpudata;
625
626 if (!policy)
627 return;
628
629 cpudata = policy->driver_data;
1d215f03 630
febab20c
WK		 631 if (policy->min != cpudata->min_limit_freq || policy->max != cpudata->max_limit_freq)
632 amd_pstate_update_min_max_limit(policy);
633
634
1d215f03
HR		 635 cap_perf = READ_ONCE(cpudata->highest_perf);
636 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
637
638 des_perf = cap_perf;
639 if (target_perf < capacity)
640 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
641
b26ffbf8 642 min_perf = READ_ONCE(cpudata->lowest_perf);
1d215f03
HR		 643 if (_min_perf < capacity)
644 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
645
646 if (min_perf < lowest_nonlinear_perf)
647 min_perf = lowest_nonlinear_perf;
648
649 max_perf = cap_perf;
650 if (max_perf < min_perf)
651 max_perf = min_perf;
652
3bf8c630 653 des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
3bf8c630 654
2dd6d0eb
WK		 655 amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true,
656 policy->governor->flags);
4f3085f8 657 cpufreq_cpu_put(policy);
1d215f03
HR		 658}
659
c8c68c38 660static int amd_pstate_cpu_boost_update(struct cpufreq_policy *policy, bool on)
41271016
HR		 661{
662 struct amd_cpudata *cpudata = policy->driver_data;
c8c68c38
PY		 663 struct cppc_perf_ctrls perf_ctrls;
664 u32 highest_perf, nominal_perf, nominal_freq, max_freq;
67d95303 665 int ret = 0;
41271016 666
c8c68c38
PY		 667 highest_perf = READ_ONCE(cpudata->highest_perf);
668 nominal_perf = READ_ONCE(cpudata->nominal_perf);
669 nominal_freq = READ_ONCE(cpudata->nominal_freq);
670 max_freq = READ_ONCE(cpudata->max_freq);
671
672 if (boot_cpu_has(X86_FEATURE_CPPC)) {
673 u64 value = READ_ONCE(cpudata->cppc_req_cached);
674
675 value &= ~GENMASK_ULL(7, 0);
676 value |= on ? highest_perf : nominal_perf;
677 WRITE_ONCE(cpudata->cppc_req_cached, value);
678
679 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
680 } else {
681 perf_ctrls.max_perf = on ? highest_perf : nominal_perf;
682 ret = cppc_set_perf(cpudata->cpu, &perf_ctrls);
683 if (ret) {
684 cpufreq_cpu_release(policy);
685 pr_debug("Failed to set max perf on CPU:%d. ret:%d\n",
686 cpudata->cpu, ret);
687 return ret;
688 }
41271016
HR		 689 }
690
c8c68c38
PY		 691 if (on)
692 policy->cpuinfo.max_freq = max_freq;
693 else if (policy->cpuinfo.max_freq > nominal_freq * 1000)
694 policy->cpuinfo.max_freq = nominal_freq * 1000;
41271016
HR		 695
696 policy->max = policy->cpuinfo.max_freq;
697
c8c68c38
PY		 698 if (cppc_state == AMD_PSTATE_PASSIVE) {
699 ret = freq_qos_update_request(&cpudata->req[1], policy->cpuinfo.max_freq);
700 if (ret < 0)
701 pr_debug("Failed to update freq constraint: CPU%d\n", cpudata->cpu);
702 }
41271016 703
c8c68c38 704 return ret < 0 ? ret : 0;
41271016
HR		 705}
706
c8c68c38 707static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
41271016 708{
c8c68c38
PY		 709 struct amd_cpudata *cpudata = policy->driver_data;
710 int ret;
41271016 711
c8c68c38
PY		 712 if (!cpudata->boost_supported) {
713 pr_err("Boost mode is not supported by this processor or SBIOS\n");
714 return -EOPNOTSUPP;
715 }
716 mutex_lock(&amd_pstate_driver_lock);
717 ret = amd_pstate_cpu_boost_update(policy, state);
718 WRITE_ONCE(cpudata->boost_state, !ret ? state : false);
719 policy->boost_enabled = !ret ? state : false;
720 refresh_frequency_limits(policy);
721 mutex_unlock(&amd_pstate_driver_lock);
41271016 722
c8c68c38
PY		 723 return ret;
724}
725
726static int amd_pstate_init_boost_support(struct amd_cpudata *cpudata)
727{
728 u64 boost_val;
729 int ret = -1;
41271016 730
c8c68c38
PY		 731 /*
732 * If platform has no CPB support or disable it, initialize current driver
733 * boost_enabled state to be false, it is not an error for cpufreq core to handle.
734 */
735 if (!cpu_feature_enabled(X86_FEATURE_CPB)) {
736 pr_debug_once("Boost CPB capabilities not present in the processor\n");
737 ret = 0;
738 goto exit_err;
739 }
740
741 /* at least one CPU supports CPB, even if others fail later on to set up */
ffa5096a 742 current_pstate_driver->boost_enabled = true;
c8c68c38
PY		 743
744 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_K7_HWCR, &boost_val);
745 if (ret) {
746 pr_err_once("failed to read initial CPU boost state!\n");
747 ret = -EIO;
748 goto exit_err;
749 }
750
751 if (!(boost_val & MSR_K7_HWCR_CPB_DIS))
752 cpudata->boost_supported = true;
753
754 return 0;
755
756exit_err:
757 cpudata->boost_supported = false;
758 return ret;
41271016
HR		 759}
760
919f4557
WK		 761static void amd_perf_ctl_reset(unsigned int cpu)
762{
763 wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
764}
765
f3a05239
ML		 766/*
767 * Set amd-pstate preferred core enable can't be done directly from cpufreq callbacks
768 * due to locking, so queue the work for later.
769 */
770static void amd_pstste_sched_prefcore_workfn(struct work_struct *work)
771{
772 sched_set_itmt_support();
773}
774static DECLARE_WORK(sched_prefcore_work, amd_pstste_sched_prefcore_workfn);
775
f3a05239
ML		 776#define CPPC_MAX_PERF U8_MAX
777
778static void amd_pstate_init_prefcore(struct amd_cpudata *cpudata)
779{
279f838a
ML		 780 /* user disabled or not detected */
781 if (!amd_pstate_prefcore)
f3a05239
ML		 782 return;
783
784 cpudata->hw_prefcore = true;
f3a05239
ML		 785
786 /*
787 * The priorities can be set regardless of whether or not
788 * sched_set_itmt_support(true) has been called and it is valid to
789 * update them at any time after it has been called.
790 */
279f838a 791 sched_set_itmt_core_prio((int)READ_ONCE(cpudata->highest_perf), cpudata->cpu);
f3a05239
ML		 792
793 schedule_work(&sched_prefcore_work);
794}
795
e571a5e2
ML		 796static void amd_pstate_update_limits(unsigned int cpu)
797{
798 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
5493f971 799 struct amd_cpudata *cpudata;
e571a5e2
ML		 800 u32 prev_high = 0, cur_high = 0;
801 int ret;
802 bool highest_perf_changed = false;
803
5493f971
AB		 804 if (!policy)
805 return;
806
807 cpudata = policy->driver_data;
808
45722e77
ML		 809 if (!amd_pstate_prefcore)
810 return;
e571a5e2 811
45722e77 812 mutex_lock(&amd_pstate_driver_lock);
2819bfef 813 ret = amd_get_highest_perf(cpu, &cur_high);
e571a5e2
ML		 814 if (ret)
815 goto free_cpufreq_put;
816
817 prev_high = READ_ONCE(cpudata->prefcore_ranking);
45722e77
ML		 818 highest_perf_changed = (prev_high != cur_high);
819 if (highest_perf_changed) {
e571a5e2
ML		 820 WRITE_ONCE(cpudata->prefcore_ranking, cur_high);
821
822 if (cur_high < CPPC_MAX_PERF)
823 sched_set_itmt_core_prio((int)cur_high, cpu);
824 }
825
826free_cpufreq_put:
827 cpufreq_cpu_put(policy);
828
829 if (!highest_perf_changed)
830 cpufreq_update_policy(cpu);
831
832 mutex_unlock(&amd_pstate_driver_lock);
833}
834
5131a3ca 835/*
069a2bb8
PY		 836 * Get pstate transition delay time from ACPI tables that firmware set
837 * instead of using hardcode value directly.
838 */
839static u32 amd_pstate_get_transition_delay_us(unsigned int cpu)
840{
841 u32 transition_delay_ns;
842
843 transition_delay_ns = cppc_get_transition_latency(cpu);
c00d476c
XD		 844 if (transition_delay_ns == CPUFREQ_ETERNAL) {
845 if (cpu_feature_enabled(X86_FEATURE_FAST_CPPC))
846 return AMD_PSTATE_FAST_CPPC_TRANSITION_DELAY;
847 else
848 return AMD_PSTATE_TRANSITION_DELAY;
849 }
069a2bb8
PY		 850
851 return transition_delay_ns / NSEC_PER_USEC;
852}
853
5131a3ca 854/*
069a2bb8
PY		 855 * Get pstate transition latency value from ACPI tables that firmware
856 * set instead of using hardcode value directly.
857 */
858static u32 amd_pstate_get_transition_latency(unsigned int cpu)
859{
860 u32 transition_latency;
861
862 transition_latency = cppc_get_transition_latency(cpu);
863 if (transition_latency == CPUFREQ_ETERNAL)
864 return AMD_PSTATE_TRANSITION_LATENCY;
865
866 return transition_latency;
867}
868
5131a3ca 869/*
5547c0eb
PY		 870 * amd_pstate_init_freq: Initialize the max_freq, min_freq,
871 * nominal_freq and lowest_nonlinear_freq for
872 * the @cpudata object.
873 *
874 * Requires: highest_perf, lowest_perf, nominal_perf and
875 * lowest_nonlinear_perf members of @cpudata to be
876 * initialized.
877 *
878 * Returns 0 on success, non-zero value on failure.
879 */
880static int amd_pstate_init_freq(struct amd_cpudata *cpudata)
881{
882 int ret;
ad4caad5
ML		 883 u32 min_freq, max_freq;
884 u64 numerator;
5547c0eb
PY		 885 u32 nominal_perf, nominal_freq;
886 u32 lowest_nonlinear_perf, lowest_nonlinear_freq;
887 u32 boost_ratio, lowest_nonlinear_ratio;
888 struct cppc_perf_caps cppc_perf;
889
5547c0eb
PY		 890 ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
891 if (ret)
892 return ret;
893
eb8b6c36
PY		 894 if (quirks && quirks->lowest_freq)
895 min_freq = quirks->lowest_freq * 1000;
896 else
897 min_freq = cppc_perf.lowest_freq * 1000;
898
899 if (quirks && quirks->nominal_freq)
900 nominal_freq = quirks->nominal_freq ;
901 else
902 nominal_freq = cppc_perf.nominal_freq;
903
5547c0eb
PY		 904 nominal_perf = READ_ONCE(cpudata->nominal_perf);
905
ad4caad5
ML		 906 ret = amd_get_boost_ratio_numerator(cpudata->cpu, &numerator);
907 if (ret)
908 return ret;
909 boost_ratio = div_u64(numerator << SCHED_CAPACITY_SHIFT, nominal_perf);
5547c0eb
PY		 910 max_freq = (nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT) * 1000;
911
912 lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
913 lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
914 nominal_perf);
915 lowest_nonlinear_freq = (nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT) * 1000;
916
917 WRITE_ONCE(cpudata->min_freq, min_freq);
918 WRITE_ONCE(cpudata->lowest_nonlinear_freq, lowest_nonlinear_freq);
919 WRITE_ONCE(cpudata->nominal_freq, nominal_freq);
920 WRITE_ONCE(cpudata->max_freq, max_freq);
921
8f8b42c1
PY		 922 /**
923 * Below values need to be initialized correctly, otherwise driver will fail to load
924 * max_freq is calculated according to (nominal_freq * highest_perf)/nominal_perf
925 * lowest_nonlinear_freq is a value between [min_freq, nominal_freq]
926 * Check _CPC in ACPI table objects if any values are incorrect
927 */
928 if (min_freq <= 0 || max_freq <= 0 || nominal_freq <= 0 || min_freq > max_freq) {
929 pr_err("min_freq(%d) or max_freq(%d) or nominal_freq(%d) value is incorrect\n",
930 min_freq, max_freq, nominal_freq * 1000);
931 return -EINVAL;
932 }
933
934 if (lowest_nonlinear_freq <= min_freq || lowest_nonlinear_freq > nominal_freq * 1000) {
935 pr_err("lowest_nonlinear_freq(%d) value is out of range [min_freq(%d), nominal_freq(%d)]\n",
936 lowest_nonlinear_freq, min_freq, nominal_freq * 1000);
937 return -EINVAL;
938 }
939
5547c0eb
PY		 940 return 0;
941}
942
ec437d71
HR		 943static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
944{
7bf7f229 945 int min_freq, max_freq, ret;
ec437d71
HR		 946 struct device *dev;
947 struct amd_cpudata *cpudata;
948
919f4557
WK		 949 /*
950 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
951 * which is ideal for initialization process.
952 */
953 amd_perf_ctl_reset(policy->cpu);
ec437d71
HR		 954 dev = get_cpu_device(policy->cpu);
955 if (!dev)
956 return -ENODEV;
957
958 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
959 if (!cpudata)
960 return -ENOMEM;
961
962 cpudata->cpu = policy->cpu;
963
964 ret = amd_pstate_init_perf(cpudata);
965 if (ret)
41271016 966 goto free_cpudata1;
ec437d71 967
279f838a
ML		 968 amd_pstate_init_prefcore(cpudata);
969
5547c0eb
PY		 970 ret = amd_pstate_init_freq(cpudata);
971 if (ret)
972 goto free_cpudata1;
973
c8c68c38
PY		 974 ret = amd_pstate_init_boost_support(cpudata);
975 if (ret)
976 goto free_cpudata1;
977
3cbbe887
GS		 978 min_freq = READ_ONCE(cpudata->min_freq);
979 max_freq = READ_ONCE(cpudata->max_freq);
ec437d71 980
069a2bb8
PY		 981 policy->cpuinfo.transition_latency = amd_pstate_get_transition_latency(policy->cpu);
982 policy->transition_delay_us = amd_pstate_get_transition_delay_us(policy->cpu);
ec437d71
HR		 983
984 policy->min = min_freq;
985 policy->max = max_freq;
986
987 policy->cpuinfo.min_freq = min_freq;
988 policy->cpuinfo.max_freq = max_freq;
989
c8c68c38
PY		 990 policy->boost_enabled = READ_ONCE(cpudata->boost_supported);
991
ec437d71
HR		 992 /* It will be updated by governor */
993 policy->cur = policy->cpuinfo.min_freq;
994
c9fdaba8 995 if (cpu_feature_enabled(X86_FEATURE_CPPC))
e059c184 996 policy->fast_switch_possible = true;
1d215f03 997
41271016
HR		 998 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
999 FREQ_QOS_MIN, policy->cpuinfo.min_freq);
1000 if (ret < 0) {
1001 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
1002 goto free_cpudata1;
1003 }
1004
1005 ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
1006 FREQ_QOS_MAX, policy->cpuinfo.max_freq);
1007 if (ret < 0) {
1008 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
1009 goto free_cpudata2;
1010 }
1011
febab20c
WK		 1012 cpudata->max_limit_freq = max_freq;
1013 cpudata->min_limit_freq = min_freq;
ec437d71
HR		 1014
1015 policy->driver_data = cpudata;
1016
abd61c08
PY		 1017 if (!current_pstate_driver->adjust_perf)
1018 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
41271016 1019
ec437d71
HR		 1020 return 0;
1021
41271016
HR		 1022free_cpudata2:
1023 freq_qos_remove_request(&cpudata->req[0]);
1024free_cpudata1:
ec437d71
HR		 1025 kfree(cpudata);
1026 return ret;
1027}
1028
b4b1ddc9 1029static void amd_pstate_cpu_exit(struct cpufreq_policy *policy)
ec437d71 1030{
4f59540c 1031 struct amd_cpudata *cpudata = policy->driver_data;
ec437d71 1032
41271016
HR		 1033 freq_qos_remove_request(&cpudata->req[1]);
1034 freq_qos_remove_request(&cpudata->req[0]);
4badf2eb 1035 policy->fast_switch_possible = false;
ec437d71 1036 kfree(cpudata);
ec437d71
HR		 1037}
1038
b376471f
JS		 1039static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
1040{
1041 int ret;
1042
1043 ret = amd_pstate_enable(true);
1044 if (ret)
1045 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
1046
1047 return ret;
1048}
1049
1050static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
1051{
1052 int ret;
1053
1054 ret = amd_pstate_enable(false);
1055 if (ret)
1056 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
1057
1058 return ret;
1059}
1060
ec4e3326
HR		 1061/* Sysfs attributes */
1062
1063/*
1064 * This frequency is to indicate the maximum hardware frequency.
1065 * If boost is not active but supported, the frequency will be larger than the
1066 * one in cpuinfo.
1067 */
1068static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
1069 char *buf)
1070{
1071 int max_freq;
4f59540c 1072 struct amd_cpudata *cpudata = policy->driver_data;
ec4e3326 1073
3cbbe887 1074 max_freq = READ_ONCE(cpudata->max_freq);
ec4e3326
HR		 1075 if (max_freq < 0)
1076 return max_freq;
1077
3ec32b6d 1078 return sysfs_emit(buf, "%u\n", max_freq);
ec4e3326
HR		 1079}
1080
1081static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
1082 char *buf)
1083{
1084 int freq;
4f59540c 1085 struct amd_cpudata *cpudata = policy->driver_data;
ec4e3326 1086
3cbbe887 1087 freq = READ_ONCE(cpudata->lowest_nonlinear_freq);
ec4e3326
HR		 1088 if (freq < 0)
1089 return freq;
1090
3ec32b6d 1091 return sysfs_emit(buf, "%u\n", freq);
ec4e3326
HR		 1092}
1093
3ad7fde1
HR		 1094/*
1095 * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
1096 * need to expose it to sysfs.
1097 */
1098static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
1099 char *buf)
1100{
1101 u32 perf;
1102 struct amd_cpudata *cpudata = policy->driver_data;
1103
1104 perf = READ_ONCE(cpudata->highest_perf);
1105
3ec32b6d 1106 return sysfs_emit(buf, "%u\n", perf);
3ad7fde1
HR		 1107}
1108
e571a5e2
ML		 1109static ssize_t show_amd_pstate_prefcore_ranking(struct cpufreq_policy *policy,
1110 char *buf)
1111{
1112 u32 perf;
1113 struct amd_cpudata *cpudata = policy->driver_data;
1114
1115 perf = READ_ONCE(cpudata->prefcore_ranking);
1116
1117 return sysfs_emit(buf, "%u\n", perf);
1118}
1119
f3a05239
ML		 1120static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy,
1121 char *buf)
1122{
1123 bool hw_prefcore;
1124 struct amd_cpudata *cpudata = policy->driver_data;
1125
1126 hw_prefcore = READ_ONCE(cpudata->hw_prefcore);
1127
1128 return sysfs_emit(buf, "%s\n", str_enabled_disabled(hw_prefcore));
1129}
1130
ffa5096a
PY		 1131static ssize_t show_energy_performance_available_preferences(
1132 struct cpufreq_policy *policy, char *buf)
1133{
1134 int i = 0;
1135 int offset = 0;
142c169b
AJ		 1136 struct amd_cpudata *cpudata = policy->driver_data;
1137
1138 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1139 return sysfs_emit_at(buf, offset, "%s\n",
1140 energy_perf_strings[EPP_INDEX_PERFORMANCE]);
ffa5096a
PY		 1141
1142 while (energy_perf_strings[i] != NULL)
1143 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]);
1144
142c169b 1145 offset += sysfs_emit_at(buf, offset, "\n");
ffa5096a
PY		 1146
1147 return offset;
1148}
1149
1150static ssize_t store_energy_performance_preference(
1151 struct cpufreq_policy *policy, const char *buf, size_t count)
1152{
1153 struct amd_cpudata *cpudata = policy->driver_data;
1154 char str_preference[21];
1155 ssize_t ret;
1156
1157 ret = sscanf(buf, "%20s", str_preference);
1158 if (ret != 1)
1159 return -EINVAL;
1160
1161 ret = match_string(energy_perf_strings, -1, str_preference);
1162 if (ret < 0)
1163 return -EINVAL;
1164
1165 mutex_lock(&amd_pstate_limits_lock);
1166 ret = amd_pstate_set_energy_pref_index(cpudata, ret);
1167 mutex_unlock(&amd_pstate_limits_lock);
1168
1169 return ret ?: count;
1170}
1171
1172static ssize_t show_energy_performance_preference(
1173 struct cpufreq_policy *policy, char *buf)
1174{
1175 struct amd_cpudata *cpudata = policy->driver_data;
1176 int preference;
1177
1178 preference = amd_pstate_get_energy_pref_index(cpudata);
1179 if (preference < 0)
1180 return preference;
1181
1182 return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
1183}
1184
3ca7bc81
WK		 1185static void amd_pstate_driver_cleanup(void)
1186{
1187 amd_pstate_enable(false);
1188 cppc_state = AMD_PSTATE_DISABLE;
1189 current_pstate_driver = NULL;
1190}
1191
1192static int amd_pstate_register_driver(int mode)
1193{
1194 int ret;
1195
1196 if (mode == AMD_PSTATE_PASSIVE || mode == AMD_PSTATE_GUIDED)
1197 current_pstate_driver = &amd_pstate_driver;
1198 else if (mode == AMD_PSTATE_ACTIVE)
1199 current_pstate_driver = &amd_pstate_epp_driver;
1200 else
1201 return -EINVAL;
1202
1203 cppc_state = mode;
1204 ret = cpufreq_register_driver(current_pstate_driver);
1205 if (ret) {
1206 amd_pstate_driver_cleanup();
1207 return ret;
1208 }
1209 return 0;
1210}
1211
1212static int amd_pstate_unregister_driver(int dummy)
1213{
1214 cpufreq_unregister_driver(current_pstate_driver);
1215 amd_pstate_driver_cleanup();
1216 return 0;
1217}
1218
1219static int amd_pstate_change_mode_without_dvr_change(int mode)
1220{
1221 int cpu = 0;
1222
1223 cppc_state = mode;
1224
c9fdaba8 1225 if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE)
3ca7bc81
WK		 1226 return 0;
1227
1228 for_each_present_cpu(cpu) {
1229 cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1);
1230 }
1231
1232 return 0;
1233}
1234
1235static int amd_pstate_change_driver_mode(int mode)
1236{
1237 int ret;
1238
1239 ret = amd_pstate_unregister_driver(0);
1240 if (ret)
1241 return ret;
1242
1243 ret = amd_pstate_register_driver(mode);
1244 if (ret)
1245 return ret;
1246
1247 return 0;
1248}
1249
11fa52fe 1250static cppc_mode_transition_fn mode_state_machine[AMD_PSTATE_MAX][AMD_PSTATE_MAX] = {
3ca7bc81
WK		 1251 [AMD_PSTATE_DISABLE] = {
1252 [AMD_PSTATE_DISABLE] = NULL,
1253 [AMD_PSTATE_PASSIVE] = amd_pstate_register_driver,
1254 [AMD_PSTATE_ACTIVE] = amd_pstate_register_driver,
1255 [AMD_PSTATE_GUIDED] = amd_pstate_register_driver,
1256 },
1257 [AMD_PSTATE_PASSIVE] = {
1258 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1259 [AMD_PSTATE_PASSIVE] = NULL,
1260 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
1261 [AMD_PSTATE_GUIDED] = amd_pstate_change_mode_without_dvr_change,
1262 },
1263 [AMD_PSTATE_ACTIVE] = {
1264 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1265 [AMD_PSTATE_PASSIVE] = amd_pstate_change_driver_mode,
1266 [AMD_PSTATE_ACTIVE] = NULL,
1267 [AMD_PSTATE_GUIDED] = amd_pstate_change_driver_mode,
1268 },
1269 [AMD_PSTATE_GUIDED] = {
1270 [AMD_PSTATE_DISABLE] = amd_pstate_unregister_driver,
1271 [AMD_PSTATE_PASSIVE] = amd_pstate_change_mode_without_dvr_change,
1272 [AMD_PSTATE_ACTIVE] = amd_pstate_change_driver_mode,
1273 [AMD_PSTATE_GUIDED] = NULL,
1274 },
1275};
1276
abd61c08
PY		 1277static ssize_t amd_pstate_show_status(char *buf)
1278{
1279 if (!current_pstate_driver)
1280 return sysfs_emit(buf, "disable\n");
1281
1282 return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
1283}
1284
8d916815 1285int amd_pstate_update_status(const char *buf, size_t size)
abd61c08 1286{
abd61c08
PY		 1287 int mode_idx;
1288
3ca7bc81 1289 if (size > strlen("passive") || size < strlen("active"))
abd61c08 1290 return -EINVAL;
abd61c08 1291
3ca7bc81 1292 mode_idx = get_mode_idx_from_str(buf, size);
abd61c08 1293
3ca7bc81
WK		 1294 if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX)
1295 return -EINVAL;
abd61c08 1296
3ca7bc81
WK		 1297 if (mode_state_machine[cppc_state][mode_idx])
1298 return mode_state_machine[cppc_state][mode_idx](mode_idx);
abd61c08 1299
3ca7bc81 1300 return 0;
abd61c08 1301}
8d916815 1302EXPORT_SYMBOL_GPL(amd_pstate_update_status);
abd61c08 1303
5e720f8c
TW		 1304static ssize_t status_show(struct device *dev,
1305 struct device_attribute *attr, char *buf)
abd61c08
PY		 1306{
1307 ssize_t ret;
1308
1309 mutex_lock(&amd_pstate_driver_lock);
1310 ret = amd_pstate_show_status(buf);
1311 mutex_unlock(&amd_pstate_driver_lock);
1312
1313 return ret;
1314}
1315
5e720f8c 1316static ssize_t status_store(struct device *a, struct device_attribute *b,
abd61c08
PY		 1317 const char *buf, size_t count)
1318{
1319 char *p = memchr(buf, '\n', count);
1320 int ret;
1321
1322 mutex_lock(&amd_pstate_driver_lock);
1323 ret = amd_pstate_update_status(buf, p ? p - buf : count);
1324 mutex_unlock(&amd_pstate_driver_lock);
1325
1326 return ret < 0 ? ret : count;
1327}
1328
f3a05239
ML		 1329static ssize_t prefcore_show(struct device *dev,
1330 struct device_attribute *attr, char *buf)
1331{
1332 return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore));
1333}
1334
ec4e3326
HR		 1335cpufreq_freq_attr_ro(amd_pstate_max_freq);
1336cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
1337
3ad7fde1 1338cpufreq_freq_attr_ro(amd_pstate_highest_perf);
e571a5e2 1339cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking);
f3a05239 1340cpufreq_freq_attr_ro(amd_pstate_hw_prefcore);
ffa5096a
PY		 1341cpufreq_freq_attr_rw(energy_performance_preference);
1342cpufreq_freq_attr_ro(energy_performance_available_preferences);
5e720f8c 1343static DEVICE_ATTR_RW(status);
f3a05239 1344static DEVICE_ATTR_RO(prefcore);
3ad7fde1 1345
ec4e3326
HR		 1346static struct freq_attr *amd_pstate_attr[] = {
1347 &amd_pstate_max_freq,
1348 &amd_pstate_lowest_nonlinear_freq,
3ad7fde1 1349 &amd_pstate_highest_perf,
e571a5e2 1350 &amd_pstate_prefcore_ranking,
f3a05239 1351 &amd_pstate_hw_prefcore,
ec4e3326
HR		 1352 NULL,
1353};
1354
ffa5096a
PY		 1355static struct freq_attr *amd_pstate_epp_attr[] = {
1356 &amd_pstate_max_freq,
1357 &amd_pstate_lowest_nonlinear_freq,
1358 &amd_pstate_highest_perf,
e571a5e2 1359 &amd_pstate_prefcore_ranking,
f3a05239 1360 &amd_pstate_hw_prefcore,
ffa5096a
PY		 1361 &energy_performance_preference,
1362 &energy_performance_available_preferences,
1363 NULL,
1364};
1365
abd61c08 1366static struct attribute *pstate_global_attributes[] = {
5e720f8c 1367 &dev_attr_status.attr,
f3a05239 1368 &dev_attr_prefcore.attr,
abd61c08
PY		 1369 NULL
1370};
1371
1372static const struct attribute_group amd_pstate_global_attr_group = {
3666062b 1373 .name = "amd_pstate",
abd61c08
PY		 1374 .attrs = pstate_global_attributes,
1375};
1376
32f80b9a
ML		 1377static bool amd_pstate_acpi_pm_profile_server(void)
1378{
1379 switch (acpi_gbl_FADT.preferred_profile) {
1380 case PM_ENTERPRISE_SERVER:
1381 case PM_SOHO_SERVER:
1382 case PM_PERFORMANCE_SERVER:
1383 return true;
1384 }
1385 return false;
1386}
1387
1388static bool amd_pstate_acpi_pm_profile_undefined(void)
1389{
1390 if (acpi_gbl_FADT.preferred_profile == PM_UNSPECIFIED)
1391 return true;
1392 if (acpi_gbl_FADT.preferred_profile >= NR_PM_PROFILES)
1393 return true;
1394 return false;
1395}
1396
ffa5096a
PY		 1397static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
1398{
7bf7f229 1399 int min_freq, max_freq, ret;
ffa5096a
PY		 1400 struct amd_cpudata *cpudata;
1401 struct device *dev;
ffa5096a
PY		 1402 u64 value;
1403
1404 /*
1405 * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
1406 * which is ideal for initialization process.
1407 */
1408 amd_perf_ctl_reset(policy->cpu);
1409 dev = get_cpu_device(policy->cpu);
1410 if (!dev)
7cca9a98 1411 return -ENODEV;
ffa5096a
PY		 1412
1413 cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
1414 if (!cpudata)
1415 return -ENOMEM;
1416
1417 cpudata->cpu = policy->cpu;
1418 cpudata->epp_policy = 0;
1419
7cca9a98
AB		 1420 ret = amd_pstate_init_perf(cpudata);
1421 if (ret)
ffa5096a
PY		 1422 goto free_cpudata1;
1423
279f838a
ML		 1424 amd_pstate_init_prefcore(cpudata);
1425
5547c0eb
PY		 1426 ret = amd_pstate_init_freq(cpudata);
1427 if (ret)
1428 goto free_cpudata1;
1429
c8c68c38
PY		 1430 ret = amd_pstate_init_boost_support(cpudata);
1431 if (ret)
1432 goto free_cpudata1;
1433
3cbbe887
GS		 1434 min_freq = READ_ONCE(cpudata->min_freq);
1435 max_freq = READ_ONCE(cpudata->max_freq);
ffa5096a
PY		 1436
1437 policy->cpuinfo.min_freq = min_freq;
1438 policy->cpuinfo.max_freq = max_freq;
1439 /* It will be updated by governor */
1440 policy->cur = policy->cpuinfo.min_freq;
1441
ffa5096a
PY		 1442 policy->driver_data = cpudata;
1443
fc6e0837 1444 cpudata->epp_cached = cpudata->epp_default = amd_pstate_get_epp(cpudata, 0);
ffa5096a
PY		 1445
1446 policy->min = policy->cpuinfo.min_freq;
1447 policy->max = policy->cpuinfo.max_freq;
1448
c8c68c38
PY		 1449 policy->boost_enabled = READ_ONCE(cpudata->boost_supported);
1450
ffa5096a 1451 /*
32f80b9a 1452 * Set the policy to provide a valid fallback value in case
ffa5096a
PY		 1453 * the default cpufreq governor is neither powersave nor performance.
1454 */
32f80b9a
ML		 1455 if (amd_pstate_acpi_pm_profile_server() ||
1456 amd_pstate_acpi_pm_profile_undefined())
1457 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
1458 else
1459 policy->policy = CPUFREQ_POLICY_POWERSAVE;
ffa5096a 1460
c9fdaba8 1461 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
ffa5096a
PY		 1462 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
1463 if (ret)
1464 return ret;
1465 WRITE_ONCE(cpudata->cppc_req_cached, value);
1466
1467 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);
1468 if (ret)
1469 return ret;
1470 WRITE_ONCE(cpudata->cppc_cap1_cached, value);
1471 }
ffa5096a
PY		 1472
1473 return 0;
1474
1475free_cpudata1:
1476 kfree(cpudata);
1477 return ret;
1478}
1479
b4b1ddc9 1480static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
ffa5096a 1481{
cea04f3d
PM		 1482 struct amd_cpudata *cpudata = policy->driver_data;
1483
1484 if (cpudata) {
1485 kfree(cpudata);
1486 policy->driver_data = NULL;
1487 }
1488
ffa5096a 1489 pr_debug("CPU %d exiting\n", policy->cpu);
ffa5096a
PY		 1490}
1491
c3e093ef 1492static int amd_pstate_epp_update_limit(struct cpufreq_policy *policy)
ffa5096a 1493{
ffa5096a 1494 struct amd_cpudata *cpudata = policy->driver_data;
febab20c 1495 u32 max_perf, min_perf, min_limit_perf, max_limit_perf;
ffa5096a
PY		 1496 u64 value;
1497 s16 epp;
1498
1499 max_perf = READ_ONCE(cpudata->highest_perf);
1500 min_perf = READ_ONCE(cpudata->lowest_perf);
febab20c
WK		 1501 max_limit_perf = div_u64(policy->max * cpudata->highest_perf, cpudata->max_freq);
1502 min_limit_perf = div_u64(policy->min * cpudata->highest_perf, cpudata->max_freq);
1503
8164f743
ML		 1504 if (min_limit_perf < min_perf)
1505 min_limit_perf = min_perf;
1506
1507 if (max_limit_perf < min_limit_perf)
1508 max_limit_perf = min_limit_perf;
1509
22fb4f04
ML		 1510 WRITE_ONCE(cpudata->max_limit_perf, max_limit_perf);
1511 WRITE_ONCE(cpudata->min_limit_perf, min_limit_perf);
1512
febab20c
WK		 1513 max_perf = clamp_t(unsigned long, max_perf, cpudata->min_limit_perf,
1514 cpudata->max_limit_perf);
1515 min_perf = clamp_t(unsigned long, min_perf, cpudata->min_limit_perf,
1516 cpudata->max_limit_perf);
ffa5096a
PY		 1517 value = READ_ONCE(cpudata->cppc_req_cached);
1518
1519 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1520 min_perf = max_perf;
1521
1522 /* Initial min/max values for CPPC Performance Controls Register */
1523 value &= ~AMD_CPPC_MIN_PERF(~0L);
1524 value |= AMD_CPPC_MIN_PERF(min_perf);
1525
1526 value &= ~AMD_CPPC_MAX_PERF(~0L);
1527 value |= AMD_CPPC_MAX_PERF(max_perf);
1528
1529 /* CPPC EPP feature require to set zero to the desire perf bit */
1530 value &= ~AMD_CPPC_DES_PERF(~0L);
1531 value |= AMD_CPPC_DES_PERF(0);
1532
ffa5096a
PY		 1533 cpudata->epp_policy = cpudata->policy;
1534
6e9d1212
WK		 1535 /* Get BIOS pre-defined epp value */
1536 epp = amd_pstate_get_epp(cpudata, value);
1537 if (epp < 0) {
1538 /**
1539 * This return value can only be negative for shared_memory
1540 * systems where EPP register read/write not supported.
1541 */
c3e093ef 1542 return epp;
ffa5096a 1543 }
6e9d1212
WK		 1544
1545 if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1546 epp = 0;
1547
ffa5096a 1548 /* Set initial EPP value */
c9fdaba8 1549 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
ffa5096a
PY		 1550 value &= ~GENMASK_ULL(31, 24);
1551 value |= (u64)epp << 24;
1552 }
1553
6e9d1212 1554 WRITE_ONCE(cpudata->cppc_req_cached, value);
c3e093ef 1555 return amd_pstate_set_epp(cpudata, epp);
ffa5096a
PY		 1556}
1557
1558static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
1559{
1560 struct amd_cpudata *cpudata = policy->driver_data;
c3e093ef 1561 int ret;
ffa5096a
PY		 1562
1563 if (!policy->cpuinfo.max_freq)
1564 return -ENODEV;
1565
1566 pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
1567 policy->cpuinfo.max_freq, policy->max);
1568
1569 cpudata->policy = policy->policy;
1570
c3e093ef
ML		 1571 ret = amd_pstate_epp_update_limit(policy);
1572 if (ret)
1573 return ret;
ffa5096a 1574
e8f555da
ML		 1575 /*
1576 * policy->cur is never updated with the amd_pstate_epp driver, but it
1577 * is used as a stale frequency value. So, keep it within limits.
1578 */
1579 policy->cur = policy->min;
1580
ffa5096a
PY		 1581 return 0;
1582}
1583
d4da12f8
PY		 1584static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)
1585{
1586 struct cppc_perf_ctrls perf_ctrls;
1587 u64 value, max_perf;
1588 int ret;
1589
1590 ret = amd_pstate_enable(true);
1591 if (ret)
1592 pr_err("failed to enable amd pstate during resume, return %d\n", ret);
1593
1594 value = READ_ONCE(cpudata->cppc_req_cached);
1595 max_perf = READ_ONCE(cpudata->highest_perf);
1596
c9fdaba8 1597 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
d4da12f8
PY		 1598 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1599 } else {
1600 perf_ctrls.max_perf = max_perf;
1601 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);
1602 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1603 }
1604}
1605
1606static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)
1607{
1608 struct amd_cpudata *cpudata = policy->driver_data;
1609
1610 pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);
1611
1612 if (cppc_state == AMD_PSTATE_ACTIVE) {
1613 amd_pstate_epp_reenable(cpudata);
1614 cpudata->suspended = false;
1615 }
1616
1617 return 0;
1618}
1619
1620static void amd_pstate_epp_offline(struct cpufreq_policy *policy)
1621{
1622 struct amd_cpudata *cpudata = policy->driver_data;
1623 struct cppc_perf_ctrls perf_ctrls;
1624 int min_perf;
1625 u64 value;
1626
1627 min_perf = READ_ONCE(cpudata->lowest_perf);
1628 value = READ_ONCE(cpudata->cppc_req_cached);
1629
1630 mutex_lock(&amd_pstate_limits_lock);
c9fdaba8 1631 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
d4da12f8
PY		 1632 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;
1633
1634 /* Set max perf same as min perf */
1635 value &= ~AMD_CPPC_MAX_PERF(~0L);
1636 value |= AMD_CPPC_MAX_PERF(min_perf);
1637 value &= ~AMD_CPPC_MIN_PERF(~0L);
1638 value |= AMD_CPPC_MIN_PERF(min_perf);
1639 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1640 } else {
1641 perf_ctrls.desired_perf = 0;
1642 perf_ctrls.max_perf = min_perf;
1643 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE);
1644 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1645 }
1646 mutex_unlock(&amd_pstate_limits_lock);
1647}
1648
1649static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)
1650{
1651 struct amd_cpudata *cpudata = policy->driver_data;
1652
1653 pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);
1654
1655 if (cpudata->suspended)
1656 return 0;
1657
1658 if (cppc_state == AMD_PSTATE_ACTIVE)
1659 amd_pstate_epp_offline(policy);
1660
1661 return 0;
1662}
1663
ffa5096a
PY		 1664static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy)
1665{
1666 cpufreq_verify_within_cpu_limits(policy);
1667 pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min);
1668 return 0;
1669}
1670
50ddd2f7
PY		 1671static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)
1672{
1673 struct amd_cpudata *cpudata = policy->driver_data;
1674 int ret;
1675
1676 /* avoid suspending when EPP is not enabled */
1677 if (cppc_state != AMD_PSTATE_ACTIVE)
1678 return 0;
1679
1680 /* set this flag to avoid setting core offline*/
1681 cpudata->suspended = true;
1682
1683 /* disable CPPC in lowlevel firmware */
1684 ret = amd_pstate_enable(false);
1685 if (ret)
1686 pr_err("failed to suspend, return %d\n", ret);
1687
1688 return 0;
1689}
1690
1691static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
1692{
1693 struct amd_cpudata *cpudata = policy->driver_data;
1694
1695 if (cpudata->suspended) {
1696 mutex_lock(&amd_pstate_limits_lock);
1697
1698 /* enable amd pstate from suspend state*/
1699 amd_pstate_epp_reenable(cpudata);
1700
1701 mutex_unlock(&amd_pstate_limits_lock);
1702
1703 cpudata->suspended = false;
1704 }
1705
1706 return 0;
1707}
1708
ec437d71
HR		 1709static struct cpufreq_driver amd_pstate_driver = {
1710 .flags = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
1711 .verify = amd_pstate_verify,
1712 .target = amd_pstate_target,
4badf2eb 1713 .fast_switch = amd_pstate_fast_switch,
ec437d71
HR		 1714 .init = amd_pstate_cpu_init,
1715 .exit = amd_pstate_cpu_exit,
b376471f
JS		 1716 .suspend = amd_pstate_cpu_suspend,
1717 .resume = amd_pstate_cpu_resume,
41271016 1718 .set_boost = amd_pstate_set_boost,
e571a5e2 1719 .update_limits = amd_pstate_update_limits,
ec437d71 1720 .name = "amd-pstate",
d8bee41d 1721 .attr = amd_pstate_attr,
ec437d71
HR		 1722};
1723
ffa5096a
PY		 1724static struct cpufreq_driver amd_pstate_epp_driver = {
1725 .flags = CPUFREQ_CONST_LOOPS,
1726 .verify = amd_pstate_epp_verify_policy,
1727 .setpolicy = amd_pstate_epp_set_policy,
1728 .init = amd_pstate_epp_cpu_init,
1729 .exit = amd_pstate_epp_cpu_exit,
d4da12f8
PY		 1730 .offline = amd_pstate_epp_cpu_offline,
1731 .online = amd_pstate_epp_cpu_online,
50ddd2f7
PY		 1732 .suspend = amd_pstate_epp_suspend,
1733 .resume = amd_pstate_epp_resume,
e571a5e2 1734 .update_limits = amd_pstate_update_limits,
c8c68c38 1735 .set_boost = amd_pstate_set_boost,
f4aad639 1736 .name = "amd-pstate-epp",
ffa5096a
PY		 1737 .attr = amd_pstate_epp_attr,
1738};
1739
c88ad30e
ML		 1740static int __init amd_pstate_set_driver(int mode_idx)
1741{
1742 if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
1743 cppc_state = mode_idx;
1744 if (cppc_state == AMD_PSTATE_DISABLE)
1745 pr_info("driver is explicitly disabled\n");
1746
1747 if (cppc_state == AMD_PSTATE_ACTIVE)
1748 current_pstate_driver = &amd_pstate_epp_driver;
1749
1750 if (cppc_state == AMD_PSTATE_PASSIVE || cppc_state == AMD_PSTATE_GUIDED)
1751 current_pstate_driver = &amd_pstate_driver;
1752
1753 return 0;
1754 }
1755
1756 return -EINVAL;
1757}
1758
cb817ec6
PY		 1759/**
1760 * CPPC function is not supported for family ID 17H with model_ID ranging from 0x10 to 0x2F.
1761 * show the debug message that helps to check if the CPU has CPPC support for loading issue.
1762 */
1763static bool amd_cppc_supported(void)
1764{
bff7d13c
PY		 1765 struct cpuinfo_x86 *c = &cpu_data(0);
1766 bool warn = false;
1767
cb817ec6
PY		 1768 if ((boot_cpu_data.x86 == 0x17) && (boot_cpu_data.x86_model < 0x30)) {
1769 pr_debug_once("CPPC feature is not supported by the processor\n");
1770 return false;
1771 }
1772
bff7d13c 1773 /*
9c68a3b0
GS		 1774 * If the CPPC feature is disabled in the BIOS for processors
1775 * that support MSR-based CPPC, the AMD Pstate driver may not
1776 * function correctly.
1777 *
1778 * For such processors, check the CPPC flag and display a
1779 * warning message if the platform supports CPPC.
1780 *
1781 * Note: The code check below will not abort the driver
1782 * registration process because of the code is added for
1783 * debugging purposes. Besides, it may still be possible for
1784 * the driver to work using the shared-memory mechanism.
bff7d13c
PY		 1785 */
1786 if (!cpu_feature_enabled(X86_FEATURE_CPPC)) {
9c68a3b0
GS		 1787 if (cpu_feature_enabled(X86_FEATURE_ZEN2)) {
1788 switch (c->x86_model) {
1789 case 0x60 ... 0x6F:
1790 case 0x80 ... 0xAF:
bff7d13c 1791 warn = true;
9c68a3b0
GS		 1792 break;
1793 }
1794 } else if (cpu_feature_enabled(X86_FEATURE_ZEN3) ||
1795 cpu_feature_enabled(X86_FEATURE_ZEN4)) {
1796 switch (c->x86_model) {
1797 case 0x10 ... 0x1F:
1798 case 0x40 ... 0xAF:
bff7d13c 1799 warn = true;
9c68a3b0
GS		 1800 break;
1801 }
bff7d13c
PY		 1802 } else if (cpu_feature_enabled(X86_FEATURE_ZEN5)) {
1803 warn = true;
1804 }
1805 }
1806
1807 if (warn)
1808 pr_warn_once("The CPPC feature is supported but currently disabled by the BIOS.\n"
1809 "Please enable it if your BIOS has the CPPC option.\n");
cb817ec6
PY		 1810 return true;
1811}
1812
ec437d71
HR		 1813static int __init amd_pstate_init(void)
1814{
3666062b 1815 struct device *dev_root;
ec437d71
HR		 1816 int ret;
1817
1818 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
1819 return -ENODEV;
1820
cb817ec6
PY		 1821 /* show debug message only if CPPC is not supported */
1822 if (!amd_cppc_supported())
1823 return -EOPNOTSUPP;
1824
1825 /* show warning message when BIOS broken or ACPI disabled */
ec437d71 1826 if (!acpi_cpc_valid()) {
a2a9d185 1827 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
ec437d71
HR		 1828 return -ENODEV;
1829 }
1830
1831 /* don't keep reloading if cpufreq_driver exists */
1832 if (cpufreq_get_current_driver())
1833 return -EEXIST;
1834
eb8b6c36
PY		 1835 quirks = NULL;
1836
1837 /* check if this machine need CPPC quirks */
1838 dmi_check_system(amd_pstate_quirks_table);
1839
4e4f600e
PY		 1840 /*
1841 * determine the driver mode from the command line or kernel config.
1842 * If no command line input is provided, cppc_state will be AMD_PSTATE_UNDEFINED.
1843 * command line options will override the kernel config settings.
1844 */
1845
1846 if (cppc_state == AMD_PSTATE_UNDEFINED) {
c88ad30e
ML		 1847 /* Disable on the following configs by default:
1848 * 1. Undefined platforms
1849 * 2. Server platforms
c88ad30e
ML		 1850 */
1851 if (amd_pstate_acpi_pm_profile_undefined() ||
91826393 1852 amd_pstate_acpi_pm_profile_server()) {
c88ad30e
ML		 1853 pr_info("driver load is disabled, boot with specific mode to enable this\n");
1854 return -ENODEV;
1855 }
4e4f600e
PY		 1856 /* get driver mode from kernel config option [1:4] */
1857 cppc_state = CONFIG_X86_AMD_PSTATE_DEFAULT_MODE;
1858 }
1859
1860 switch (cppc_state) {
c88ad30e 1861 case AMD_PSTATE_DISABLE:
4e4f600e 1862 pr_info("driver load is disabled, boot with specific mode to enable this\n");
c88ad30e
ML		 1863 return -ENODEV;
1864 case AMD_PSTATE_PASSIVE:
1865 case AMD_PSTATE_ACTIVE:
1866 case AMD_PSTATE_GUIDED:
4e4f600e
PY		 1867 ret = amd_pstate_set_driver(cppc_state);
1868 if (ret)
1869 return ret;
c88ad30e
ML		 1870 break;
1871 default:
1872 return -EINVAL;
1873 }
1874
ec437d71 1875 /* capability check */
c9fdaba8 1876 if (cpu_feature_enabled(X86_FEATURE_CPPC)) {
e059c184 1877 pr_debug("AMD CPPC MSR based functionality is supported\n");
2dd6d0eb 1878 if (cppc_state != AMD_PSTATE_ACTIVE)
ffa5096a 1879 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
202e683d
PY		 1880 } else {
1881 pr_debug("AMD CPPC shared memory based functionality is supported\n");
e059c184
HR		 1882 static_call_update(amd_pstate_enable, cppc_enable);
1883 static_call_update(amd_pstate_init_perf, cppc_init_perf);
1884 static_call_update(amd_pstate_update_perf, cppc_update_perf);
ec437d71
HR		 1885 }
1886
279f838a
ML		 1887 if (amd_pstate_prefcore) {
1888 ret = amd_detect_prefcore(&amd_pstate_prefcore);
1889 if (ret)
1890 return ret;
1891 }
1892
ec437d71
HR		 1893 /* enable amd pstate feature */
1894 ret = amd_pstate_enable(true);
1895 if (ret) {
4e4f600e 1896 pr_err("failed to enable driver mode(%d)\n", cppc_state);
ec437d71
HR		 1897 return ret;
1898 }
1899
ffa5096a 1900 ret = cpufreq_register_driver(current_pstate_driver);
bc76f575 1901 if (ret) {
ffa5096a 1902 pr_err("failed to register with return %d\n", ret);
bc76f575
ML		 1903 goto disable_driver;
1904 }
ec437d71 1905
3666062b
GKH		 1906 dev_root = bus_get_dev_root(&cpu_subsys);
1907 if (dev_root) {
1908 ret = sysfs_create_group(&dev_root->kobj, &amd_pstate_global_attr_group);
1909 put_device(dev_root);
1910 if (ret) {
1911 pr_err("sysfs attribute export failed with error %d.\n", ret);
1912 goto global_attr_free;
1913 }
abd61c08
PY		 1914 }
1915
1916 return ret;
1917
1918global_attr_free:
abd61c08 1919 cpufreq_unregister_driver(current_pstate_driver);
bc76f575
ML		 1920disable_driver:
1921 amd_pstate_enable(false);
ec437d71
HR		 1922 return ret;
1923}
456ca88d 1924device_initcall(amd_pstate_init);
ec437d71 1925
202e683d
PY		 1926static int __init amd_pstate_param(char *str)
1927{
36c5014e
WK		 1928 size_t size;
1929 int mode_idx;
1930
202e683d
PY		 1931 if (!str)
1932 return -EINVAL;
1933
36c5014e
WK		 1934 size = strlen(str);
1935 mode_idx = get_mode_idx_from_str(str, size);
202e683d 1936
c88ad30e 1937 return amd_pstate_set_driver(mode_idx);
202e683d 1938}
f3a05239
ML		 1939
1940static int __init amd_prefcore_param(char *str)
1941{
1942 if (!strcmp(str, "disable"))
1943 amd_pstate_prefcore = false;
1944
1945 return 0;
1946}
1947
202e683d 1948early_param("amd_pstate", amd_pstate_param);
f3a05239 1949early_param("amd_prefcore", amd_prefcore_param);
202e683d 1950
ec437d71
HR		 1951MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
1952MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");
Linux 6.x block layer and io_uring tree(s)