drivers/cpufreq/amd-pstate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * amd-pstate.c - AMD Processor P-state Frequency Driver
   4  *
   5  * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
   6  *
   7  * Author: Huang Rui <ray.huang@amd.com>
   8  *
   9  * AMD P-State introduces a new CPU performance scaling design for AMD
  10  * processors using the ACPI Collaborative Performance and Power Control (CPPC)
  11  * feature which works with the AMD SMU firmware providing a finer grained
  12  * frequency control range. It is to replace the legacy ACPI P-States control,
  13  * allows a flexible, low-latency interface for the Linux kernel to directly
  14  * communicate the performance hints to hardware.
  15  *
  16  * AMD P-State is supported on recent AMD Zen base CPU series include some of
  17  * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
  18  * P-State supported system. And there are two types of hardware implementations
  19  * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
  20  * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
  21  */
  22
  23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25 #include <linux/kernel.h>
  26 #include <linux/module.h>
  27 #include <linux/init.h>
  28 #include <linux/smp.h>
  29 #include <linux/sched.h>
  30 #include <linux/cpufreq.h>
  31 #include <linux/compiler.h>
  32 #include <linux/dmi.h>
  33 #include <linux/slab.h>
  34 #include <linux/acpi.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/static_call.h>
  39 #include <linux/amd-pstate.h>
  40
  41 #include <acpi/processor.h>
  42 #include <acpi/cppc_acpi.h>
  43
  44 #include <asm/msr.h>
  45 #include <asm/processor.h>
  46 #include <asm/cpufeature.h>
  47 #include <asm/cpu_device_id.h>
  48 #include "amd-pstate-trace.h"
  49
  50 #define AMD_PSTATE_TRANSITION_LATENCY   20000
  51 #define AMD_PSTATE_TRANSITION_DELAY     1000
  52
  53 /*
  54  * TODO: We need more time to fine tune processors with shared memory solution
  55  * with community together.
  56  *
  57  * There are some performance drops on the CPU benchmarks which reports from
  58  * Suse. We are co-working with them to fine tune the shared memory solution. So
  59  * we disable it by default to go acpi-cpufreq on these processors and add a
  60  * module parameter to be able to enable it manually for debugging.
  61  */
  62 static struct cpufreq_driver *current_pstate_driver;
  63 static struct cpufreq_driver amd_pstate_driver;
  64 static struct cpufreq_driver amd_pstate_epp_driver;
  65 static int cppc_state = AMD_PSTATE_DISABLE;
  66 struct kobject *amd_pstate_kobj;
  67
  68 /*
  69  * AMD Energy Preference Performance (EPP)
  70  * The EPP is used in the CCLK DPM controller to drive
  71  * the frequency that a core is going to operate during
  72  * short periods of activity. EPP values will be utilized for
  73  * different OS profiles (balanced, performance, power savings)
  74  * display strings corresponding to EPP index in the
  75  * energy_perf_strings[]
  76  *      index           String
  77  *-------------------------------------
  78  *      0               default
  79  *      1               performance
  80  *      2               balance_performance
  81  *      3               balance_power
  82  *      4               power
  83  */
  84 enum energy_perf_value_index {
  85         EPP_INDEX_DEFAULT = 0,
  86         EPP_INDEX_PERFORMANCE,
  87         EPP_INDEX_BALANCE_PERFORMANCE,
  88         EPP_INDEX_BALANCE_POWERSAVE,
  89         EPP_INDEX_POWERSAVE,
  90 };
  91
  92 static const char * const energy_perf_strings[] = {
  93         [EPP_INDEX_DEFAULT] = "default",
  94         [EPP_INDEX_PERFORMANCE] = "performance",
  95         [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
  96         [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
  97         [EPP_INDEX_POWERSAVE] = "power",
  98         NULL
  99 };
 100
 101 static unsigned int epp_values[] = {
 102         [EPP_INDEX_DEFAULT] = 0,
 103         [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
 104         [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
 105         [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
 106         [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
 107  };
 108
 109 static inline int get_mode_idx_from_str(const char *str, size_t size)
 110 {
 111         int i;
 112
 113         for (i=0; i < AMD_PSTATE_MAX; i++) {
 114                 if (!strncmp(str, amd_pstate_mode_string[i], size))
 115                         return i;
 116         }
 117         return -EINVAL;
 118 }
 119
 120 static DEFINE_MUTEX(amd_pstate_limits_lock);
 121 static DEFINE_MUTEX(amd_pstate_driver_lock);
 122
 123 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)
 124 {
 125         u64 epp;
 126         int ret;
 127
 128         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 129                 if (!cppc_req_cached) {
 130                         epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 131                                         &cppc_req_cached);
 132                         if (epp)
 133                                 return epp;
 134                 }
 135                 epp = (cppc_req_cached >> 24) & 0xFF;
 136         } else {
 137                 ret = cppc_get_epp_perf(cpudata->cpu, &epp);
 138                 if (ret < 0) {
 139                         pr_debug("Could not retrieve energy perf value (%d)\n", ret);
 140                         return -EIO;
 141                 }
 142         }
 143
 144         return (s16)(epp & 0xff);
 145 }
 146
 147 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)
 148 {
 149         s16 epp;
 150         int index = -EINVAL;
 151
 152         epp = amd_pstate_get_epp(cpudata, 0);
 153         if (epp < 0)
 154                 return epp;
 155
 156         switch (epp) {
 157         case AMD_CPPC_EPP_PERFORMANCE:
 158                 index = EPP_INDEX_PERFORMANCE;
 159                 break;
 160         case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
 161                 index = EPP_INDEX_BALANCE_PERFORMANCE;
 162                 break;
 163         case AMD_CPPC_EPP_BALANCE_POWERSAVE:
 164                 index = EPP_INDEX_BALANCE_POWERSAVE;
 165                 break;
 166         case AMD_CPPC_EPP_POWERSAVE:
 167                 index = EPP_INDEX_POWERSAVE;
 168                 break;
 169         default:
 170                 break;
 171         }
 172
 173         return index;
 174 }
 175
 176 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)
 177 {
 178         int ret;
 179         struct cppc_perf_ctrls perf_ctrls;
 180
 181         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 182                 u64 value = READ_ONCE(cpudata->cppc_req_cached);
 183
 184                 value &= ~GENMASK_ULL(31, 24);
 185                 value |= (u64)epp << 24;
 186                 WRITE_ONCE(cpudata->cppc_req_cached, value);
 187
 188                 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
 189                 if (!ret)
 190                         cpudata->epp_cached = epp;
 191         } else {
 192                 perf_ctrls.energy_perf = epp;
 193                 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
 194                 if (ret) {
 195                         pr_debug("failed to set energy perf value (%d)\n", ret);
 196                         return ret;
 197                 }
 198                 cpudata->epp_cached = epp;
 199         }
 200
 201         return ret;
 202 }
 203
 204 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,
 205                 int pref_index)
 206 {
 207         int epp = -EINVAL;
 208         int ret;
 209
 210         if (!pref_index) {
 211                 pr_debug("EPP pref_index is invalid\n");
 212                 return -EINVAL;
 213         }
 214
 215         if (epp == -EINVAL)
 216                 epp = epp_values[pref_index];
 217
 218         if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
 219                 pr_debug("EPP cannot be set under performance policy\n");
 220                 return -EBUSY;
 221         }
 222
 223         ret = amd_pstate_set_epp(cpudata, epp);
 224
 225         return ret;
 226 }
 227
 228 static inline int pstate_enable(bool enable)
 229 {
 230         return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable);
 231 }
 232
 233 static int cppc_enable(bool enable)
 234 {
 235         int cpu, ret = 0;
 236         struct cppc_perf_ctrls perf_ctrls;
 237
 238         for_each_present_cpu(cpu) {
 239                 ret = cppc_set_enable(cpu, enable);
 240                 if (ret)
 241                         return ret;
 242
 243                 /* Enable autonomous mode for EPP */
 244                 if (cppc_state == AMD_PSTATE_ACTIVE) {
 245                         /* Set desired perf as zero to allow EPP firmware control */
 246                         perf_ctrls.desired_perf = 0;
 247                         ret = cppc_set_perf(cpu, &perf_ctrls);
 248                         if (ret)
 249                                 return ret;
 250                 }
 251         }
 252
 253         return ret;
 254 }
 255
 256 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
 257
 258 static inline int amd_pstate_enable(bool enable)
 259 {
 260         return static_call(amd_pstate_enable)(enable);
 261 }
 262
 263 static int pstate_init_perf(struct amd_cpudata *cpudata)
 264 {
 265         u64 cap1;
 266         u32 highest_perf;
 267
 268         int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
 269                                      &cap1);
 270         if (ret)
 271                 return ret;
 272
 273         /*
 274          * TODO: Introduce AMD specific power feature.
 275          *
 276          * CPPC entry doesn't indicate the highest performance in some ASICs.
 277          */
 278         highest_perf = amd_get_highest_perf();
 279         if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
 280                 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
 281
 282         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 283
 284         WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
 285         WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
 286         WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
 287
 288         return 0;
 289 }
 290
 291 static int cppc_init_perf(struct amd_cpudata *cpudata)
 292 {
 293         struct cppc_perf_caps cppc_perf;
 294         u32 highest_perf;
 295
 296         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 297         if (ret)
 298                 return ret;
 299
 300         highest_perf = amd_get_highest_perf();
 301         if (highest_perf > cppc_perf.highest_perf)
 302                 highest_perf = cppc_perf.highest_perf;
 303
 304         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 305
 306         WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
 307         WRITE_ONCE(cpudata->lowest_nonlinear_perf,
 308                    cppc_perf.lowest_nonlinear_perf);
 309         WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
 310
 311         return 0;
 312 }
 313
 314 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
 315
 316 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
 317 {
 318         return static_call(amd_pstate_init_perf)(cpudata);
 319 }
 320
 321 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
 322                                u32 des_perf, u32 max_perf, bool fast_switch)
 323 {
 324         if (fast_switch)
 325                 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
 326         else
 327                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 328                               READ_ONCE(cpudata->cppc_req_cached));
 329 }
 330
 331 static void cppc_update_perf(struct amd_cpudata *cpudata,
 332                              u32 min_perf, u32 des_perf,
 333                              u32 max_perf, bool fast_switch)
 334 {
 335         struct cppc_perf_ctrls perf_ctrls;
 336
 337         perf_ctrls.max_perf = max_perf;
 338         perf_ctrls.min_perf = min_perf;
 339         perf_ctrls.desired_perf = des_perf;
 340
 341         cppc_set_perf(cpudata->cpu, &perf_ctrls);
 342 }
 343
 344 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
 345
 346 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
 347                                           u32 min_perf, u32 des_perf,
 348                                           u32 max_perf, bool fast_switch)
 349 {
 350         static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
 351                                             max_perf, fast_switch);
 352 }
 353
 354 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
 355 {
 356         u64 aperf, mperf, tsc;
 357         unsigned long flags;
 358
 359         local_irq_save(flags);
 360         rdmsrl(MSR_IA32_APERF, aperf);
 361         rdmsrl(MSR_IA32_MPERF, mperf);
 362         tsc = rdtsc();
 363
 364         if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
 365                 local_irq_restore(flags);
 366                 return false;
 367         }
 368
 369         local_irq_restore(flags);
 370
 371         cpudata->cur.aperf = aperf;
 372         cpudata->cur.mperf = mperf;
 373         cpudata->cur.tsc =  tsc;
 374         cpudata->cur.aperf -= cpudata->prev.aperf;
 375         cpudata->cur.mperf -= cpudata->prev.mperf;
 376         cpudata->cur.tsc -= cpudata->prev.tsc;
 377
 378         cpudata->prev.aperf = aperf;
 379         cpudata->prev.mperf = mperf;
 380         cpudata->prev.tsc = tsc;
 381
 382         cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
 383
 384         return true;
 385 }
 386
 387 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
 388                               u32 des_perf, u32 max_perf, bool fast_switch)
 389 {
 390         u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 391         u64 value = prev;
 392
 393         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 394         value &= ~AMD_CPPC_MIN_PERF(~0L);
 395         value |= AMD_CPPC_MIN_PERF(min_perf);
 396
 397         value &= ~AMD_CPPC_DES_PERF(~0L);
 398         value |= AMD_CPPC_DES_PERF(des_perf);
 399
 400         value &= ~AMD_CPPC_MAX_PERF(~0L);
 401         value |= AMD_CPPC_MAX_PERF(max_perf);
 402
 403         if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
 404                 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
 405                         cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
 406                                 cpudata->cpu, (value != prev), fast_switch);
 407         }
 408
 409         if (value == prev)
 410                 return;
 411
 412         WRITE_ONCE(cpudata->cppc_req_cached, value);
 413
 414         amd_pstate_update_perf(cpudata, min_perf, des_perf,
 415                                max_perf, fast_switch);
 416 }
 417
 418 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
 419 {
 420         cpufreq_verify_within_cpu_limits(policy);
 421
 422         return 0;
 423 }
 424
 425 static int amd_pstate_target(struct cpufreq_policy *policy,
 426                              unsigned int target_freq,
 427                              unsigned int relation)
 428 {
 429         struct cpufreq_freqs freqs;
 430         struct amd_cpudata *cpudata = policy->driver_data;
 431         unsigned long max_perf, min_perf, des_perf, cap_perf;
 432
 433         if (!cpudata->max_freq)
 434                 return -ENODEV;
 435
 436         cap_perf = READ_ONCE(cpudata->highest_perf);
 437         min_perf = READ_ONCE(cpudata->lowest_perf);
 438         max_perf = cap_perf;
 439
 440         freqs.old = policy->cur;
 441         freqs.new = target_freq;
 442
 443         des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
 444                                      cpudata->max_freq);
 445
 446         cpufreq_freq_transition_begin(policy, &freqs);
 447         amd_pstate_update(cpudata, min_perf, des_perf,
 448                           max_perf, false);
 449         cpufreq_freq_transition_end(policy, &freqs, false);
 450
 451         return 0;
 452 }
 453
 454 static void amd_pstate_adjust_perf(unsigned int cpu,
 455                                    unsigned long _min_perf,
 456                                    unsigned long target_perf,
 457                                    unsigned long capacity)
 458 {
 459         unsigned long max_perf, min_perf, des_perf,
 460                       cap_perf, lowest_nonlinear_perf;
 461         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 462         struct amd_cpudata *cpudata = policy->driver_data;
 463
 464         cap_perf = READ_ONCE(cpudata->highest_perf);
 465         lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
 466
 467         des_perf = cap_perf;
 468         if (target_perf < capacity)
 469                 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
 470
 471         min_perf = READ_ONCE(cpudata->highest_perf);
 472         if (_min_perf < capacity)
 473                 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
 474
 475         if (min_perf < lowest_nonlinear_perf)
 476                 min_perf = lowest_nonlinear_perf;
 477
 478         max_perf = cap_perf;
 479         if (max_perf < min_perf)
 480                 max_perf = min_perf;
 481
 482         amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true);
 483         cpufreq_cpu_put(policy);
 484 }
 485
 486 static int amd_get_min_freq(struct amd_cpudata *cpudata)
 487 {
 488         struct cppc_perf_caps cppc_perf;
 489
 490         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 491         if (ret)
 492                 return ret;
 493
 494         /* Switch to khz */
 495         return cppc_perf.lowest_freq * 1000;
 496 }
 497
 498 static int amd_get_max_freq(struct amd_cpudata *cpudata)
 499 {
 500         struct cppc_perf_caps cppc_perf;
 501         u32 max_perf, max_freq, nominal_freq, nominal_perf;
 502         u64 boost_ratio;
 503
 504         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 505         if (ret)
 506                 return ret;
 507
 508         nominal_freq = cppc_perf.nominal_freq;
 509         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 510         max_perf = READ_ONCE(cpudata->highest_perf);
 511
 512         boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
 513                               nominal_perf);
 514
 515         max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
 516
 517         /* Switch to khz */
 518         return max_freq * 1000;
 519 }
 520
 521 static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
 522 {
 523         struct cppc_perf_caps cppc_perf;
 524
 525         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 526         if (ret)
 527                 return ret;
 528
 529         /* Switch to khz */
 530         return cppc_perf.nominal_freq * 1000;
 531 }
 532
 533 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
 534 {
 535         struct cppc_perf_caps cppc_perf;
 536         u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
 537             nominal_freq, nominal_perf;
 538         u64 lowest_nonlinear_ratio;
 539
 540         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 541         if (ret)
 542                 return ret;
 543
 544         nominal_freq = cppc_perf.nominal_freq;
 545         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 546
 547         lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
 548
 549         lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
 550                                          nominal_perf);
 551
 552         lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
 553
 554         /* Switch to khz */
 555         return lowest_nonlinear_freq * 1000;
 556 }
 557
 558 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
 559 {
 560         struct amd_cpudata *cpudata = policy->driver_data;
 561         int ret;
 562
 563         if (!cpudata->boost_supported) {
 564                 pr_err("Boost mode is not supported by this processor or SBIOS\n");
 565                 return -EINVAL;
 566         }
 567
 568         if (state)
 569                 policy->cpuinfo.max_freq = cpudata->max_freq;
 570         else
 571                 policy->cpuinfo.max_freq = cpudata->nominal_freq;
 572
 573         policy->max = policy->cpuinfo.max_freq;
 574
 575         ret = freq_qos_update_request(&cpudata->req[1],
 576                                       policy->cpuinfo.max_freq);
 577         if (ret < 0)
 578                 return ret;
 579
 580         return 0;
 581 }
 582
 583 static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
 584 {
 585         u32 highest_perf, nominal_perf;
 586
 587         highest_perf = READ_ONCE(cpudata->highest_perf);
 588         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 589
 590         if (highest_perf <= nominal_perf)
 591                 return;
 592
 593         cpudata->boost_supported = true;
 594         current_pstate_driver->boost_enabled = true;
 595 }
 596
 597 static void amd_perf_ctl_reset(unsigned int cpu)
 598 {
 599         wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
 600 }
 601
 602 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
 603 {
 604         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
 605         struct device *dev;
 606         struct amd_cpudata *cpudata;
 607
 608         /*
 609          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
 610          * which is ideal for initialization process.
 611          */
 612         amd_perf_ctl_reset(policy->cpu);
 613         dev = get_cpu_device(policy->cpu);
 614         if (!dev)
 615                 return -ENODEV;
 616
 617         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 618         if (!cpudata)
 619                 return -ENOMEM;
 620
 621         cpudata->cpu = policy->cpu;
 622
 623         ret = amd_pstate_init_perf(cpudata);
 624         if (ret)
 625                 goto free_cpudata1;
 626
 627         min_freq = amd_get_min_freq(cpudata);
 628         max_freq = amd_get_max_freq(cpudata);
 629         nominal_freq = amd_get_nominal_freq(cpudata);
 630         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
 631
 632         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
 633                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
 634                         min_freq, max_freq);
 635                 ret = -EINVAL;
 636                 goto free_cpudata1;
 637         }
 638
 639         policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
 640         policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
 641
 642         policy->min = min_freq;
 643         policy->max = max_freq;
 644
 645         policy->cpuinfo.min_freq = min_freq;
 646         policy->cpuinfo.max_freq = max_freq;
 647
 648         /* It will be updated by governor */
 649         policy->cur = policy->cpuinfo.min_freq;
 650
 651         if (boot_cpu_has(X86_FEATURE_CPPC))
 652                 policy->fast_switch_possible = true;
 653
 654         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
 655                                    FREQ_QOS_MIN, policy->cpuinfo.min_freq);
 656         if (ret < 0) {
 657                 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
 658                 goto free_cpudata1;
 659         }
 660
 661         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
 662                                    FREQ_QOS_MAX, policy->cpuinfo.max_freq);
 663         if (ret < 0) {
 664                 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
 665                 goto free_cpudata2;
 666         }
 667
 668         /* Initial processor data capability frequencies */
 669         cpudata->max_freq = max_freq;
 670         cpudata->min_freq = min_freq;
 671         cpudata->nominal_freq = nominal_freq;
 672         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
 673
 674         policy->driver_data = cpudata;
 675
 676         amd_pstate_boost_init(cpudata);
 677         if (!current_pstate_driver->adjust_perf)
 678                 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
 679
 680         return 0;
 681
 682 free_cpudata2:
 683         freq_qos_remove_request(&cpudata->req[0]);
 684 free_cpudata1:
 685         kfree(cpudata);
 686         return ret;
 687 }
 688
 689 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
 690 {
 691         struct amd_cpudata *cpudata = policy->driver_data;
 692
 693         freq_qos_remove_request(&cpudata->req[1]);
 694         freq_qos_remove_request(&cpudata->req[0]);
 695         kfree(cpudata);
 696
 697         return 0;
 698 }
 699
 700 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
 701 {
 702         int ret;
 703
 704         ret = amd_pstate_enable(true);
 705         if (ret)
 706                 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
 707
 708         return ret;
 709 }
 710
 711 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
 712 {
 713         int ret;
 714
 715         ret = amd_pstate_enable(false);
 716         if (ret)
 717                 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
 718
 719         return ret;
 720 }
 721
 722 /* Sysfs attributes */
 723
 724 /*
 725  * This frequency is to indicate the maximum hardware frequency.
 726  * If boost is not active but supported, the frequency will be larger than the
 727  * one in cpuinfo.
 728  */
 729 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
 730                                         char *buf)
 731 {
 732         int max_freq;
 733         struct amd_cpudata *cpudata = policy->driver_data;
 734
 735         max_freq = amd_get_max_freq(cpudata);
 736         if (max_freq < 0)
 737                 return max_freq;
 738
 739         return sysfs_emit(buf, "%u\n", max_freq);
 740 }
 741
 742 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
 743                                                      char *buf)
 744 {
 745         int freq;
 746         struct amd_cpudata *cpudata = policy->driver_data;
 747
 748         freq = amd_get_lowest_nonlinear_freq(cpudata);
 749         if (freq < 0)
 750                 return freq;
 751
 752         return sysfs_emit(buf, "%u\n", freq);
 753 }
 754
 755 /*
 756  * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
 757  * need to expose it to sysfs.
 758  */
 759 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
 760                                             char *buf)
 761 {
 762         u32 perf;
 763         struct amd_cpudata *cpudata = policy->driver_data;
 764
 765         perf = READ_ONCE(cpudata->highest_perf);
 766
 767         return sysfs_emit(buf, "%u\n", perf);
 768 }
 769
 770 static ssize_t show_energy_performance_available_preferences(
 771                                 struct cpufreq_policy *policy, char *buf)
 772 {
 773         int i = 0;
 774         int offset = 0;
 775
 776         while (energy_perf_strings[i] != NULL)
 777                 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]);
 778
 779         sysfs_emit_at(buf, offset, "\n");
 780
 781         return offset;
 782 }
 783
 784 static ssize_t store_energy_performance_preference(
 785                 struct cpufreq_policy *policy, const char *buf, size_t count)
 786 {
 787         struct amd_cpudata *cpudata = policy->driver_data;
 788         char str_preference[21];
 789         ssize_t ret;
 790
 791         ret = sscanf(buf, "%20s", str_preference);
 792         if (ret != 1)
 793                 return -EINVAL;
 794
 795         ret = match_string(energy_perf_strings, -1, str_preference);
 796         if (ret < 0)
 797                 return -EINVAL;
 798
 799         mutex_lock(&amd_pstate_limits_lock);
 800         ret = amd_pstate_set_energy_pref_index(cpudata, ret);
 801         mutex_unlock(&amd_pstate_limits_lock);
 802
 803         return ret ?: count;
 804 }
 805
 806 static ssize_t show_energy_performance_preference(
 807                                 struct cpufreq_policy *policy, char *buf)
 808 {
 809         struct amd_cpudata *cpudata = policy->driver_data;
 810         int preference;
 811
 812         preference = amd_pstate_get_energy_pref_index(cpudata);
 813         if (preference < 0)
 814                 return preference;
 815
 816         return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
 817 }
 818
 819 static ssize_t amd_pstate_show_status(char *buf)
 820 {
 821         if (!current_pstate_driver)
 822                 return sysfs_emit(buf, "disable\n");
 823
 824         return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
 825 }
 826
 827 static void amd_pstate_driver_cleanup(void)
 828 {
 829         current_pstate_driver = NULL;
 830 }
 831
 832 static int amd_pstate_update_status(const char *buf, size_t size)
 833 {
 834         int ret = 0;
 835         int mode_idx;
 836
 837         if (size > 7 || size < 6)
 838                 return -EINVAL;
 839         mode_idx = get_mode_idx_from_str(buf, size);
 840
 841         switch(mode_idx) {
 842         case AMD_PSTATE_DISABLE:
 843                 if (!current_pstate_driver)
 844                         return -EINVAL;
 845                 if (cppc_state == AMD_PSTATE_ACTIVE)
 846                         return -EBUSY;
 847                 cpufreq_unregister_driver(current_pstate_driver);
 848                 amd_pstate_driver_cleanup();
 849                 break;
 850         case AMD_PSTATE_PASSIVE:
 851                 if (current_pstate_driver) {
 852                         if (current_pstate_driver == &amd_pstate_driver)
 853                                 return 0;
 854                         cpufreq_unregister_driver(current_pstate_driver);
 855                         cppc_state = AMD_PSTATE_PASSIVE;
 856                         current_pstate_driver = &amd_pstate_driver;
 857                 }
 858
 859                 ret = cpufreq_register_driver(current_pstate_driver);
 860                 break;
 861         case AMD_PSTATE_ACTIVE:
 862                 if (current_pstate_driver) {
 863                         if (current_pstate_driver == &amd_pstate_epp_driver)
 864                                 return 0;
 865                         cpufreq_unregister_driver(current_pstate_driver);
 866                         current_pstate_driver = &amd_pstate_epp_driver;
 867                         cppc_state = AMD_PSTATE_ACTIVE;
 868                 }
 869
 870                 ret = cpufreq_register_driver(current_pstate_driver);
 871                 break;
 872         default:
 873                 ret = -EINVAL;
 874                 break;
 875         }
 876
 877         return ret;
 878 }
 879
 880 static ssize_t show_status(struct kobject *kobj,
 881                            struct kobj_attribute *attr, char *buf)
 882 {
 883         ssize_t ret;
 884
 885         mutex_lock(&amd_pstate_driver_lock);
 886         ret = amd_pstate_show_status(buf);
 887         mutex_unlock(&amd_pstate_driver_lock);
 888
 889         return ret;
 890 }
 891
 892 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b,
 893                             const char *buf, size_t count)
 894 {
 895         char *p = memchr(buf, '\n', count);
 896         int ret;
 897
 898         mutex_lock(&amd_pstate_driver_lock);
 899         ret = amd_pstate_update_status(buf, p ? p - buf : count);
 900         mutex_unlock(&amd_pstate_driver_lock);
 901
 902         return ret < 0 ? ret : count;
 903 }
 904
 905 cpufreq_freq_attr_ro(amd_pstate_max_freq);
 906 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
 907
 908 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
 909 cpufreq_freq_attr_rw(energy_performance_preference);
 910 cpufreq_freq_attr_ro(energy_performance_available_preferences);
 911 define_one_global_rw(status);
 912
 913 static struct freq_attr *amd_pstate_attr[] = {
 914         &amd_pstate_max_freq,
 915         &amd_pstate_lowest_nonlinear_freq,
 916         &amd_pstate_highest_perf,
 917         NULL,
 918 };
 919
 920 static struct freq_attr *amd_pstate_epp_attr[] = {
 921         &amd_pstate_max_freq,
 922         &amd_pstate_lowest_nonlinear_freq,
 923         &amd_pstate_highest_perf,
 924         &energy_performance_preference,
 925         &energy_performance_available_preferences,
 926         NULL,
 927 };
 928
 929 static struct attribute *pstate_global_attributes[] = {
 930         &status.attr,
 931         NULL
 932 };
 933
 934 static const struct attribute_group amd_pstate_global_attr_group = {
 935         .attrs = pstate_global_attributes,
 936 };
 937
 938 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
 939 {
 940         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
 941         struct amd_cpudata *cpudata;
 942         struct device *dev;
 943         u64 value;
 944
 945         /*
 946          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
 947          * which is ideal for initialization process.
 948          */
 949         amd_perf_ctl_reset(policy->cpu);
 950         dev = get_cpu_device(policy->cpu);
 951         if (!dev)
 952                 return -ENODEV;
 953
 954         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 955         if (!cpudata)
 956                 return -ENOMEM;
 957
 958         cpudata->cpu = policy->cpu;
 959         cpudata->epp_policy = 0;
 960
 961         ret = amd_pstate_init_perf(cpudata);
 962         if (ret)
 963                 goto free_cpudata1;
 964
 965         min_freq = amd_get_min_freq(cpudata);
 966         max_freq = amd_get_max_freq(cpudata);
 967         nominal_freq = amd_get_nominal_freq(cpudata);
 968         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
 969         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
 970                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
 971                                 min_freq, max_freq);
 972                 ret = -EINVAL;
 973                 goto free_cpudata1;
 974         }
 975
 976         policy->cpuinfo.min_freq = min_freq;
 977         policy->cpuinfo.max_freq = max_freq;
 978         /* It will be updated by governor */
 979         policy->cur = policy->cpuinfo.min_freq;
 980
 981         /* Initial processor data capability frequencies */
 982         cpudata->max_freq = max_freq;
 983         cpudata->min_freq = min_freq;
 984         cpudata->nominal_freq = nominal_freq;
 985         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
 986
 987         policy->driver_data = cpudata;
 988
 989         cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0);
 990
 991         policy->min = policy->cpuinfo.min_freq;
 992         policy->max = policy->cpuinfo.max_freq;
 993
 994         /*
 995          * Set the policy to powersave to provide a valid fallback value in case
 996          * the default cpufreq governor is neither powersave nor performance.
 997          */
 998         policy->policy = CPUFREQ_POLICY_POWERSAVE;
 999
1000         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1001                 policy->fast_switch_possible = true;
1002                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
1003                 if (ret)
1004                         return ret;
1005                 WRITE_ONCE(cpudata->cppc_req_cached, value);
1006
1007                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);
1008                 if (ret)
1009                         return ret;
1010                 WRITE_ONCE(cpudata->cppc_cap1_cached, value);
1011         }
1012         amd_pstate_boost_init(cpudata);
1013
1014         return 0;
1015
1016 free_cpudata1:
1017         kfree(cpudata);
1018         return ret;
1019 }
1020
1021 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
1022 {
1023         pr_debug("CPU %d exiting\n", policy->cpu);
1024         policy->fast_switch_possible = false;
1025         return 0;
1026 }
1027
1028 static void amd_pstate_epp_init(unsigned int cpu)
1029 {
1030         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1031         struct amd_cpudata *cpudata = policy->driver_data;
1032         u32 max_perf, min_perf;
1033         u64 value;
1034         s16 epp;
1035
1036         max_perf = READ_ONCE(cpudata->highest_perf);
1037         min_perf = READ_ONCE(cpudata->lowest_perf);
1038
1039         value = READ_ONCE(cpudata->cppc_req_cached);
1040
1041         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1042                 min_perf = max_perf;
1043
1044         /* Initial min/max values for CPPC Performance Controls Register */
1045         value &= ~AMD_CPPC_MIN_PERF(~0L);
1046         value |= AMD_CPPC_MIN_PERF(min_perf);
1047
1048         value &= ~AMD_CPPC_MAX_PERF(~0L);
1049         value |= AMD_CPPC_MAX_PERF(max_perf);
1050
1051         /* CPPC EPP feature require to set zero to the desire perf bit */
1052         value &= ~AMD_CPPC_DES_PERF(~0L);
1053         value |= AMD_CPPC_DES_PERF(0);
1054
1055         if (cpudata->epp_policy == cpudata->policy)
1056                 goto skip_epp;
1057
1058         cpudata->epp_policy = cpudata->policy;
1059
1060         /* Get BIOS pre-defined epp value */
1061         epp = amd_pstate_get_epp(cpudata, value);
1062         if (epp < 0) {
1063                 /**
1064                  * This return value can only be negative for shared_memory
1065                  * systems where EPP register read/write not supported.
1066                  */
1067                 goto skip_epp;
1068         }
1069
1070         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1071                 epp = 0;
1072
1073         /* Set initial EPP value */
1074         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1075                 value &= ~GENMASK_ULL(31, 24);
1076                 value |= (u64)epp << 24;
1077         }
1078
1079         WRITE_ONCE(cpudata->cppc_req_cached, value);
1080         amd_pstate_set_epp(cpudata, epp);
1081 skip_epp:
1082         cpufreq_cpu_put(policy);
1083 }
1084
1085 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
1086 {
1087         struct amd_cpudata *cpudata = policy->driver_data;
1088
1089         if (!policy->cpuinfo.max_freq)
1090                 return -ENODEV;
1091
1092         pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
1093                                 policy->cpuinfo.max_freq, policy->max);
1094
1095         cpudata->policy = policy->policy;
1096
1097         amd_pstate_epp_init(policy->cpu);
1098
1099         return 0;
1100 }
1101
1102 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)
1103 {
1104         struct cppc_perf_ctrls perf_ctrls;
1105         u64 value, max_perf;
1106         int ret;
1107
1108         ret = amd_pstate_enable(true);
1109         if (ret)
1110                 pr_err("failed to enable amd pstate during resume, return %d\n", ret);
1111
1112         value = READ_ONCE(cpudata->cppc_req_cached);
1113         max_perf = READ_ONCE(cpudata->highest_perf);
1114
1115         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1116                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1117         } else {
1118                 perf_ctrls.max_perf = max_perf;
1119                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);
1120                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1121         }
1122 }
1123
1124 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)
1125 {
1126         struct amd_cpudata *cpudata = policy->driver_data;
1127
1128         pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);
1129
1130         if (cppc_state == AMD_PSTATE_ACTIVE) {
1131                 amd_pstate_epp_reenable(cpudata);
1132                 cpudata->suspended = false;
1133         }
1134
1135         return 0;
1136 }
1137
1138 static void amd_pstate_epp_offline(struct cpufreq_policy *policy)
1139 {
1140         struct amd_cpudata *cpudata = policy->driver_data;
1141         struct cppc_perf_ctrls perf_ctrls;
1142         int min_perf;
1143         u64 value;
1144
1145         min_perf = READ_ONCE(cpudata->lowest_perf);
1146         value = READ_ONCE(cpudata->cppc_req_cached);
1147
1148         mutex_lock(&amd_pstate_limits_lock);
1149         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1150                 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;
1151
1152                 /* Set max perf same as min perf */
1153                 value &= ~AMD_CPPC_MAX_PERF(~0L);
1154                 value |= AMD_CPPC_MAX_PERF(min_perf);
1155                 value &= ~AMD_CPPC_MIN_PERF(~0L);
1156                 value |= AMD_CPPC_MIN_PERF(min_perf);
1157                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1158         } else {
1159                 perf_ctrls.desired_perf = 0;
1160                 perf_ctrls.max_perf = min_perf;
1161                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE);
1162                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1163         }
1164         mutex_unlock(&amd_pstate_limits_lock);
1165 }
1166
1167 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)
1168 {
1169         struct amd_cpudata *cpudata = policy->driver_data;
1170
1171         pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);
1172
1173         if (cpudata->suspended)
1174                 return 0;
1175
1176         if (cppc_state == AMD_PSTATE_ACTIVE)
1177                 amd_pstate_epp_offline(policy);
1178
1179         return 0;
1180 }
1181
1182 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy)
1183 {
1184         cpufreq_verify_within_cpu_limits(policy);
1185         pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min);
1186         return 0;
1187 }
1188
1189 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)
1190 {
1191         struct amd_cpudata *cpudata = policy->driver_data;
1192         int ret;
1193
1194         /* avoid suspending when EPP is not enabled */
1195         if (cppc_state != AMD_PSTATE_ACTIVE)
1196                 return 0;
1197
1198         /* set this flag to avoid setting core offline*/
1199         cpudata->suspended = true;
1200
1201         /* disable CPPC in lowlevel firmware */
1202         ret = amd_pstate_enable(false);
1203         if (ret)
1204                 pr_err("failed to suspend, return %d\n", ret);
1205
1206         return 0;
1207 }
1208
1209 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
1210 {
1211         struct amd_cpudata *cpudata = policy->driver_data;
1212
1213         if (cpudata->suspended) {
1214                 mutex_lock(&amd_pstate_limits_lock);
1215
1216                 /* enable amd pstate from suspend state*/
1217                 amd_pstate_epp_reenable(cpudata);
1218
1219                 mutex_unlock(&amd_pstate_limits_lock);
1220
1221                 cpudata->suspended = false;
1222         }
1223
1224         return 0;
1225 }
1226
1227 static struct cpufreq_driver amd_pstate_driver = {
1228         .flags          = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
1229         .verify         = amd_pstate_verify,
1230         .target         = amd_pstate_target,
1231         .init           = amd_pstate_cpu_init,
1232         .exit           = amd_pstate_cpu_exit,
1233         .suspend        = amd_pstate_cpu_suspend,
1234         .resume         = amd_pstate_cpu_resume,
1235         .set_boost      = amd_pstate_set_boost,
1236         .name           = "amd-pstate",
1237         .attr           = amd_pstate_attr,
1238 };
1239
1240 static struct cpufreq_driver amd_pstate_epp_driver = {
1241         .flags          = CPUFREQ_CONST_LOOPS,
1242         .verify         = amd_pstate_epp_verify_policy,
1243         .setpolicy      = amd_pstate_epp_set_policy,
1244         .init           = amd_pstate_epp_cpu_init,
1245         .exit           = amd_pstate_epp_cpu_exit,
1246         .offline        = amd_pstate_epp_cpu_offline,
1247         .online         = amd_pstate_epp_cpu_online,
1248         .suspend        = amd_pstate_epp_suspend,
1249         .resume         = amd_pstate_epp_resume,
1250         .name           = "amd_pstate_epp",
1251         .attr           = amd_pstate_epp_attr,
1252 };
1253
1254 static int __init amd_pstate_init(void)
1255 {
1256         int ret;
1257
1258         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
1259                 return -ENODEV;
1260         /*
1261          * by default the pstate driver is disabled to load
1262          * enable the amd_pstate passive mode driver explicitly
1263          * with amd_pstate=passive or other modes in kernel command line
1264          */
1265         if (cppc_state == AMD_PSTATE_DISABLE) {
1266                 pr_info("driver load is disabled, boot with specific mode to enable this\n");
1267                 return -ENODEV;
1268         }
1269
1270         if (!acpi_cpc_valid()) {
1271                 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
1272                 return -ENODEV;
1273         }
1274
1275         /* don't keep reloading if cpufreq_driver exists */
1276         if (cpufreq_get_current_driver())
1277                 return -EEXIST;
1278
1279         /* capability check */
1280         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1281                 pr_debug("AMD CPPC MSR based functionality is supported\n");
1282                 if (cppc_state == AMD_PSTATE_PASSIVE)
1283                         current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
1284         } else {
1285                 pr_debug("AMD CPPC shared memory based functionality is supported\n");
1286                 static_call_update(amd_pstate_enable, cppc_enable);
1287                 static_call_update(amd_pstate_init_perf, cppc_init_perf);
1288                 static_call_update(amd_pstate_update_perf, cppc_update_perf);
1289         }
1290
1291         /* enable amd pstate feature */
1292         ret = amd_pstate_enable(true);
1293         if (ret) {
1294                 pr_err("failed to enable with return %d\n", ret);
1295                 return ret;
1296         }
1297
1298         ret = cpufreq_register_driver(current_pstate_driver);
1299         if (ret)
1300                 pr_err("failed to register with return %d\n", ret);
1301
1302         amd_pstate_kobj = kobject_create_and_add("amd_pstate", &cpu_subsys.dev_root->kobj);
1303         if (!amd_pstate_kobj) {
1304                 ret = -EINVAL;
1305                 pr_err("global sysfs registration failed.\n");
1306                 goto kobject_free;
1307         }
1308
1309         ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group);
1310         if (ret) {
1311                 pr_err("sysfs attribute export failed with error %d.\n", ret);
1312                 goto global_attr_free;
1313         }
1314
1315         return ret;
1316
1317 global_attr_free:
1318         kobject_put(amd_pstate_kobj);
1319 kobject_free:
1320         cpufreq_unregister_driver(current_pstate_driver);
1321         return ret;
1322 }
1323 device_initcall(amd_pstate_init);
1324
1325 static int __init amd_pstate_param(char *str)
1326 {
1327         size_t size;
1328         int mode_idx;
1329
1330         if (!str)
1331                 return -EINVAL;
1332
1333         size = strlen(str);
1334         mode_idx = get_mode_idx_from_str(str, size);
1335
1336         if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
1337                 cppc_state = mode_idx;
1338                 if (cppc_state == AMD_PSTATE_DISABLE)
1339                         pr_info("driver is explicitly disabled\n");
1340
1341                 if (cppc_state == AMD_PSTATE_ACTIVE)
1342                         current_pstate_driver = &amd_pstate_epp_driver;
1343
1344                 if (cppc_state == AMD_PSTATE_PASSIVE)
1345                         current_pstate_driver = &amd_pstate_driver;
1346
1347                 return 0;
1348         }
1349
1350         return -EINVAL;
1351 }
1352 early_param("amd_pstate", amd_pstate_param);
1353
1354 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
1355 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");