drivers/cpufreq/amd-pstate.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * amd-pstate.c - AMD Processor P-state Frequency Driver
   4  *
   5  * Copyright (C) 2021 Advanced Micro Devices, Inc. All Rights Reserved.
   6  *
   7  * Author: Huang Rui <ray.huang@amd.com>
   8  *
   9  * AMD P-State introduces a new CPU performance scaling design for AMD
  10  * processors using the ACPI Collaborative Performance and Power Control (CPPC)
  11  * feature which works with the AMD SMU firmware providing a finer grained
  12  * frequency control range. It is to replace the legacy ACPI P-States control,
  13  * allows a flexible, low-latency interface for the Linux kernel to directly
  14  * communicate the performance hints to hardware.
  15  *
  16  * AMD P-State is supported on recent AMD Zen base CPU series include some of
  17  * Zen2 and Zen3 processors. _CPC needs to be present in the ACPI tables of AMD
  18  * P-State supported system. And there are two types of hardware implementations
  19  * for AMD P-State: 1) Full MSR Solution and 2) Shared Memory Solution.
  20  * X86_FEATURE_CPPC CPU feature flag is used to distinguish the different types.
  21  */
  22
  23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25 #include <linux/kernel.h>
  26 #include <linux/module.h>
  27 #include <linux/init.h>
  28 #include <linux/smp.h>
  29 #include <linux/sched.h>
  30 #include <linux/cpufreq.h>
  31 #include <linux/compiler.h>
  32 #include <linux/dmi.h>
  33 #include <linux/slab.h>
  34 #include <linux/acpi.h>
  35 #include <linux/io.h>
  36 #include <linux/delay.h>
  37 #include <linux/uaccess.h>
  38 #include <linux/static_call.h>
  39 #include <linux/amd-pstate.h>
  40
  41 #include <acpi/processor.h>
  42 #include <acpi/cppc_acpi.h>
  43
  44 #include <asm/msr.h>
  45 #include <asm/processor.h>
  46 #include <asm/cpufeature.h>
  47 #include <asm/cpu_device_id.h>
  48 #include "amd-pstate-trace.h"
  49
  50 #define AMD_PSTATE_TRANSITION_LATENCY   20000
  51 #define AMD_PSTATE_TRANSITION_DELAY     1000
  52
  53 /*
  54  * TODO: We need more time to fine tune processors with shared memory solution
  55  * with community together.
  56  *
  57  * There are some performance drops on the CPU benchmarks which reports from
  58  * Suse. We are co-working with them to fine tune the shared memory solution. So
  59  * we disable it by default to go acpi-cpufreq on these processors and add a
  60  * module parameter to be able to enable it manually for debugging.
  61  */
  62 static struct cpufreq_driver *current_pstate_driver;
  63 static struct cpufreq_driver amd_pstate_driver;
  64 static struct cpufreq_driver amd_pstate_epp_driver;
  65 static int cppc_state = AMD_PSTATE_DISABLE;
  66 struct kobject *amd_pstate_kobj;
  67
  68 /*
  69  * AMD Energy Preference Performance (EPP)
  70  * The EPP is used in the CCLK DPM controller to drive
  71  * the frequency that a core is going to operate during
  72  * short periods of activity. EPP values will be utilized for
  73  * different OS profiles (balanced, performance, power savings)
  74  * display strings corresponding to EPP index in the
  75  * energy_perf_strings[]
  76  *      index           String
  77  *-------------------------------------
  78  *      0               default
  79  *      1               performance
  80  *      2               balance_performance
  81  *      3               balance_power
  82  *      4               power
  83  */
  84 enum energy_perf_value_index {
  85         EPP_INDEX_DEFAULT = 0,
  86         EPP_INDEX_PERFORMANCE,
  87         EPP_INDEX_BALANCE_PERFORMANCE,
  88         EPP_INDEX_BALANCE_POWERSAVE,
  89         EPP_INDEX_POWERSAVE,
  90 };
  91
  92 static const char * const energy_perf_strings[] = {
  93         [EPP_INDEX_DEFAULT] = "default",
  94         [EPP_INDEX_PERFORMANCE] = "performance",
  95         [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance",
  96         [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power",
  97         [EPP_INDEX_POWERSAVE] = "power",
  98         NULL
  99 };
 100
 101 static unsigned int epp_values[] = {
 102         [EPP_INDEX_DEFAULT] = 0,
 103         [EPP_INDEX_PERFORMANCE] = AMD_CPPC_EPP_PERFORMANCE,
 104         [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE,
 105         [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE,
 106         [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE,
 107  };
 108
 109 static inline int get_mode_idx_from_str(const char *str, size_t size)
 110 {
 111         int i;
 112
 113         for (i=0; i < AMD_PSTATE_MAX; i++) {
 114                 if (!strncmp(str, amd_pstate_mode_string[i], size))
 115                         return i;
 116         }
 117         return -EINVAL;
 118 }
 119
 120 static DEFINE_MUTEX(amd_pstate_limits_lock);
 121 static DEFINE_MUTEX(amd_pstate_driver_lock);
 122
 123 static s16 amd_pstate_get_epp(struct amd_cpudata *cpudata, u64 cppc_req_cached)
 124 {
 125         u64 epp;
 126         int ret;
 127
 128         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 129                 if (!cppc_req_cached) {
 130                         epp = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 131                                         &cppc_req_cached);
 132                         if (epp)
 133                                 return epp;
 134                 }
 135                 epp = (cppc_req_cached >> 24) & 0xFF;
 136         } else {
 137                 ret = cppc_get_epp_perf(cpudata->cpu, &epp);
 138                 if (ret < 0) {
 139                         pr_debug("Could not retrieve energy perf value (%d)\n", ret);
 140                         return -EIO;
 141                 }
 142         }
 143
 144         return (s16)(epp & 0xff);
 145 }
 146
 147 static int amd_pstate_get_energy_pref_index(struct amd_cpudata *cpudata)
 148 {
 149         s16 epp;
 150         int index = -EINVAL;
 151
 152         epp = amd_pstate_get_epp(cpudata, 0);
 153         if (epp < 0)
 154                 return epp;
 155
 156         switch (epp) {
 157         case AMD_CPPC_EPP_PERFORMANCE:
 158                 index = EPP_INDEX_PERFORMANCE;
 159                 break;
 160         case AMD_CPPC_EPP_BALANCE_PERFORMANCE:
 161                 index = EPP_INDEX_BALANCE_PERFORMANCE;
 162                 break;
 163         case AMD_CPPC_EPP_BALANCE_POWERSAVE:
 164                 index = EPP_INDEX_BALANCE_POWERSAVE;
 165                 break;
 166         case AMD_CPPC_EPP_POWERSAVE:
 167                 index = EPP_INDEX_POWERSAVE;
 168                 break;
 169         default:
 170                 break;
 171         }
 172
 173         return index;
 174 }
 175
 176 static int amd_pstate_set_epp(struct amd_cpudata *cpudata, u32 epp)
 177 {
 178         int ret;
 179         struct cppc_perf_ctrls perf_ctrls;
 180
 181         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 182                 u64 value = READ_ONCE(cpudata->cppc_req_cached);
 183
 184                 value &= ~GENMASK_ULL(31, 24);
 185                 value |= (u64)epp << 24;
 186                 WRITE_ONCE(cpudata->cppc_req_cached, value);
 187
 188                 ret = wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
 189                 if (!ret)
 190                         cpudata->epp_cached = epp;
 191         } else {
 192                 perf_ctrls.energy_perf = epp;
 193                 ret = cppc_set_epp_perf(cpudata->cpu, &perf_ctrls, 1);
 194                 if (ret) {
 195                         pr_debug("failed to set energy perf value (%d)\n", ret);
 196                         return ret;
 197                 }
 198                 cpudata->epp_cached = epp;
 199         }
 200
 201         return ret;
 202 }
 203
 204 static int amd_pstate_set_energy_pref_index(struct amd_cpudata *cpudata,
 205                 int pref_index)
 206 {
 207         int epp = -EINVAL;
 208         int ret;
 209
 210         if (!pref_index) {
 211                 pr_debug("EPP pref_index is invalid\n");
 212                 return -EINVAL;
 213         }
 214
 215         if (epp == -EINVAL)
 216                 epp = epp_values[pref_index];
 217
 218         if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
 219                 pr_debug("EPP cannot be set under performance policy\n");
 220                 return -EBUSY;
 221         }
 222
 223         ret = amd_pstate_set_epp(cpudata, epp);
 224
 225         return ret;
 226 }
 227
 228 static inline int pstate_enable(bool enable)
 229 {
 230         return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable);
 231 }
 232
 233 static int cppc_enable(bool enable)
 234 {
 235         int cpu, ret = 0;
 236         struct cppc_perf_ctrls perf_ctrls;
 237
 238         for_each_present_cpu(cpu) {
 239                 ret = cppc_set_enable(cpu, enable);
 240                 if (ret)
 241                         return ret;
 242
 243                 /* Enable autonomous mode for EPP */
 244                 if (cppc_state == AMD_PSTATE_ACTIVE) {
 245                         /* Set desired perf as zero to allow EPP firmware control */
 246                         perf_ctrls.desired_perf = 0;
 247                         ret = cppc_set_perf(cpu, &perf_ctrls);
 248                         if (ret)
 249                                 return ret;
 250                 }
 251         }
 252
 253         return ret;
 254 }
 255
 256 DEFINE_STATIC_CALL(amd_pstate_enable, pstate_enable);
 257
 258 static inline int amd_pstate_enable(bool enable)
 259 {
 260         return static_call(amd_pstate_enable)(enable);
 261 }
 262
 263 static int pstate_init_perf(struct amd_cpudata *cpudata)
 264 {
 265         u64 cap1;
 266         u32 highest_perf;
 267
 268         int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
 269                                      &cap1);
 270         if (ret)
 271                 return ret;
 272
 273         /*
 274          * TODO: Introduce AMD specific power feature.
 275          *
 276          * CPPC entry doesn't indicate the highest performance in some ASICs.
 277          */
 278         highest_perf = amd_get_highest_perf();
 279         if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
 280                 highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
 281
 282         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 283
 284         WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
 285         WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
 286         WRITE_ONCE(cpudata->lowest_perf, AMD_CPPC_LOWEST_PERF(cap1));
 287
 288         return 0;
 289 }
 290
 291 static int cppc_init_perf(struct amd_cpudata *cpudata)
 292 {
 293         struct cppc_perf_caps cppc_perf;
 294         u32 highest_perf;
 295
 296         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 297         if (ret)
 298                 return ret;
 299
 300         highest_perf = amd_get_highest_perf();
 301         if (highest_perf > cppc_perf.highest_perf)
 302                 highest_perf = cppc_perf.highest_perf;
 303
 304         WRITE_ONCE(cpudata->highest_perf, highest_perf);
 305
 306         WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
 307         WRITE_ONCE(cpudata->lowest_nonlinear_perf,
 308                    cppc_perf.lowest_nonlinear_perf);
 309         WRITE_ONCE(cpudata->lowest_perf, cppc_perf.lowest_perf);
 310
 311         return 0;
 312 }
 313
 314 DEFINE_STATIC_CALL(amd_pstate_init_perf, pstate_init_perf);
 315
 316 static inline int amd_pstate_init_perf(struct amd_cpudata *cpudata)
 317 {
 318         return static_call(amd_pstate_init_perf)(cpudata);
 319 }
 320
 321 static void pstate_update_perf(struct amd_cpudata *cpudata, u32 min_perf,
 322                                u32 des_perf, u32 max_perf, bool fast_switch)
 323 {
 324         if (fast_switch)
 325                 wrmsrl(MSR_AMD_CPPC_REQ, READ_ONCE(cpudata->cppc_req_cached));
 326         else
 327                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ,
 328                               READ_ONCE(cpudata->cppc_req_cached));
 329 }
 330
 331 static void cppc_update_perf(struct amd_cpudata *cpudata,
 332                              u32 min_perf, u32 des_perf,
 333                              u32 max_perf, bool fast_switch)
 334 {
 335         struct cppc_perf_ctrls perf_ctrls;
 336
 337         perf_ctrls.max_perf = max_perf;
 338         perf_ctrls.min_perf = min_perf;
 339         perf_ctrls.desired_perf = des_perf;
 340
 341         cppc_set_perf(cpudata->cpu, &perf_ctrls);
 342 }
 343
 344 DEFINE_STATIC_CALL(amd_pstate_update_perf, pstate_update_perf);
 345
 346 static inline void amd_pstate_update_perf(struct amd_cpudata *cpudata,
 347                                           u32 min_perf, u32 des_perf,
 348                                           u32 max_perf, bool fast_switch)
 349 {
 350         static_call(amd_pstate_update_perf)(cpudata, min_perf, des_perf,
 351                                             max_perf, fast_switch);
 352 }
 353
 354 static inline bool amd_pstate_sample(struct amd_cpudata *cpudata)
 355 {
 356         u64 aperf, mperf, tsc;
 357         unsigned long flags;
 358
 359         local_irq_save(flags);
 360         rdmsrl(MSR_IA32_APERF, aperf);
 361         rdmsrl(MSR_IA32_MPERF, mperf);
 362         tsc = rdtsc();
 363
 364         if (cpudata->prev.mperf == mperf || cpudata->prev.tsc == tsc) {
 365                 local_irq_restore(flags);
 366                 return false;
 367         }
 368
 369         local_irq_restore(flags);
 370
 371         cpudata->cur.aperf = aperf;
 372         cpudata->cur.mperf = mperf;
 373         cpudata->cur.tsc =  tsc;
 374         cpudata->cur.aperf -= cpudata->prev.aperf;
 375         cpudata->cur.mperf -= cpudata->prev.mperf;
 376         cpudata->cur.tsc -= cpudata->prev.tsc;
 377
 378         cpudata->prev.aperf = aperf;
 379         cpudata->prev.mperf = mperf;
 380         cpudata->prev.tsc = tsc;
 381
 382         cpudata->freq = div64_u64((cpudata->cur.aperf * cpu_khz), cpudata->cur.mperf);
 383
 384         return true;
 385 }
 386
 387 static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
 388                               u32 des_perf, u32 max_perf, bool fast_switch)
 389 {
 390         u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 391         u64 value = prev;
 392
 393         des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 394         value &= ~AMD_CPPC_MIN_PERF(~0L);
 395         value |= AMD_CPPC_MIN_PERF(min_perf);
 396
 397         value &= ~AMD_CPPC_DES_PERF(~0L);
 398         value |= AMD_CPPC_DES_PERF(des_perf);
 399
 400         value &= ~AMD_CPPC_MAX_PERF(~0L);
 401         value |= AMD_CPPC_MAX_PERF(max_perf);
 402
 403         if (trace_amd_pstate_perf_enabled() && amd_pstate_sample(cpudata)) {
 404                 trace_amd_pstate_perf(min_perf, des_perf, max_perf, cpudata->freq,
 405                         cpudata->cur.mperf, cpudata->cur.aperf, cpudata->cur.tsc,
 406                                 cpudata->cpu, (value != prev), fast_switch);
 407         }
 408
 409         if (value == prev)
 410                 return;
 411
 412         WRITE_ONCE(cpudata->cppc_req_cached, value);
 413
 414         amd_pstate_update_perf(cpudata, min_perf, des_perf,
 415                                max_perf, fast_switch);
 416 }
 417
 418 static int amd_pstate_verify(struct cpufreq_policy_data *policy)
 419 {
 420         cpufreq_verify_within_cpu_limits(policy);
 421
 422         return 0;
 423 }
 424
 425 static int amd_pstate_target(struct cpufreq_policy *policy,
 426                              unsigned int target_freq,
 427                              unsigned int relation)
 428 {
 429         struct cpufreq_freqs freqs;
 430         struct amd_cpudata *cpudata = policy->driver_data;
 431         unsigned long max_perf, min_perf, des_perf, cap_perf;
 432
 433         if (!cpudata->max_freq)
 434                 return -ENODEV;
 435
 436         cap_perf = READ_ONCE(cpudata->highest_perf);
 437         min_perf = READ_ONCE(cpudata->lowest_perf);
 438         max_perf = cap_perf;
 439
 440         freqs.old = policy->cur;
 441         freqs.new = target_freq;
 442
 443         des_perf = DIV_ROUND_CLOSEST(target_freq * cap_perf,
 444                                      cpudata->max_freq);
 445
 446         cpufreq_freq_transition_begin(policy, &freqs);
 447         amd_pstate_update(cpudata, min_perf, des_perf,
 448                           max_perf, false);
 449         cpufreq_freq_transition_end(policy, &freqs, false);
 450
 451         return 0;
 452 }
 453
 454 static void amd_pstate_adjust_perf(unsigned int cpu,
 455                                    unsigned long _min_perf,
 456                                    unsigned long target_perf,
 457                                    unsigned long capacity)
 458 {
 459         unsigned long max_perf, min_perf, des_perf,
 460                       cap_perf, lowest_nonlinear_perf;
 461         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
 462         struct amd_cpudata *cpudata = policy->driver_data;
 463
 464         cap_perf = READ_ONCE(cpudata->highest_perf);
 465         lowest_nonlinear_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
 466
 467         des_perf = cap_perf;
 468         if (target_perf < capacity)
 469                 des_perf = DIV_ROUND_UP(cap_perf * target_perf, capacity);
 470
 471         min_perf = READ_ONCE(cpudata->highest_perf);
 472         if (_min_perf < capacity)
 473                 min_perf = DIV_ROUND_UP(cap_perf * _min_perf, capacity);
 474
 475         if (min_perf < lowest_nonlinear_perf)
 476                 min_perf = lowest_nonlinear_perf;
 477
 478         max_perf = cap_perf;
 479         if (max_perf < min_perf)
 480                 max_perf = min_perf;
 481
 482         amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true);
 483         cpufreq_cpu_put(policy);
 484 }
 485
 486 static int amd_get_min_freq(struct amd_cpudata *cpudata)
 487 {
 488         struct cppc_perf_caps cppc_perf;
 489
 490         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 491         if (ret)
 492                 return ret;
 493
 494         /* Switch to khz */
 495         return cppc_perf.lowest_freq * 1000;
 496 }
 497
 498 static int amd_get_max_freq(struct amd_cpudata *cpudata)
 499 {
 500         struct cppc_perf_caps cppc_perf;
 501         u32 max_perf, max_freq, nominal_freq, nominal_perf;
 502         u64 boost_ratio;
 503
 504         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 505         if (ret)
 506                 return ret;
 507
 508         nominal_freq = cppc_perf.nominal_freq;
 509         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 510         max_perf = READ_ONCE(cpudata->highest_perf);
 511
 512         boost_ratio = div_u64(max_perf << SCHED_CAPACITY_SHIFT,
 513                               nominal_perf);
 514
 515         max_freq = nominal_freq * boost_ratio >> SCHED_CAPACITY_SHIFT;
 516
 517         /* Switch to khz */
 518         return max_freq * 1000;
 519 }
 520
 521 static int amd_get_nominal_freq(struct amd_cpudata *cpudata)
 522 {
 523         struct cppc_perf_caps cppc_perf;
 524
 525         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 526         if (ret)
 527                 return ret;
 528
 529         /* Switch to khz */
 530         return cppc_perf.nominal_freq * 1000;
 531 }
 532
 533 static int amd_get_lowest_nonlinear_freq(struct amd_cpudata *cpudata)
 534 {
 535         struct cppc_perf_caps cppc_perf;
 536         u32 lowest_nonlinear_freq, lowest_nonlinear_perf,
 537             nominal_freq, nominal_perf;
 538         u64 lowest_nonlinear_ratio;
 539
 540         int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 541         if (ret)
 542                 return ret;
 543
 544         nominal_freq = cppc_perf.nominal_freq;
 545         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 546
 547         lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
 548
 549         lowest_nonlinear_ratio = div_u64(lowest_nonlinear_perf << SCHED_CAPACITY_SHIFT,
 550                                          nominal_perf);
 551
 552         lowest_nonlinear_freq = nominal_freq * lowest_nonlinear_ratio >> SCHED_CAPACITY_SHIFT;
 553
 554         /* Switch to khz */
 555         return lowest_nonlinear_freq * 1000;
 556 }
 557
 558 static int amd_pstate_set_boost(struct cpufreq_policy *policy, int state)
 559 {
 560         struct amd_cpudata *cpudata = policy->driver_data;
 561         int ret;
 562
 563         if (!cpudata->boost_supported) {
 564                 pr_err("Boost mode is not supported by this processor or SBIOS\n");
 565                 return -EINVAL;
 566         }
 567
 568         if (state)
 569                 policy->cpuinfo.max_freq = cpudata->max_freq;
 570         else
 571                 policy->cpuinfo.max_freq = cpudata->nominal_freq;
 572
 573         policy->max = policy->cpuinfo.max_freq;
 574
 575         ret = freq_qos_update_request(&cpudata->req[1],
 576                                       policy->cpuinfo.max_freq);
 577         if (ret < 0)
 578                 return ret;
 579
 580         return 0;
 581 }
 582
 583 static void amd_pstate_boost_init(struct amd_cpudata *cpudata)
 584 {
 585         u32 highest_perf, nominal_perf;
 586
 587         highest_perf = READ_ONCE(cpudata->highest_perf);
 588         nominal_perf = READ_ONCE(cpudata->nominal_perf);
 589
 590         if (highest_perf <= nominal_perf)
 591                 return;
 592
 593         cpudata->boost_supported = true;
 594         current_pstate_driver->boost_enabled = true;
 595 }
 596
 597 static void amd_perf_ctl_reset(unsigned int cpu)
 598 {
 599         wrmsrl_on_cpu(cpu, MSR_AMD_PERF_CTL, 0);
 600 }
 601
 602 static int amd_pstate_cpu_init(struct cpufreq_policy *policy)
 603 {
 604         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
 605         struct device *dev;
 606         struct amd_cpudata *cpudata;
 607
 608         /*
 609          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
 610          * which is ideal for initialization process.
 611          */
 612         amd_perf_ctl_reset(policy->cpu);
 613         dev = get_cpu_device(policy->cpu);
 614         if (!dev)
 615                 return -ENODEV;
 616
 617         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 618         if (!cpudata)
 619                 return -ENOMEM;
 620
 621         cpudata->cpu = policy->cpu;
 622
 623         ret = amd_pstate_init_perf(cpudata);
 624         if (ret)
 625                 goto free_cpudata1;
 626
 627         min_freq = amd_get_min_freq(cpudata);
 628         max_freq = amd_get_max_freq(cpudata);
 629         nominal_freq = amd_get_nominal_freq(cpudata);
 630         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
 631
 632         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
 633                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
 634                         min_freq, max_freq);
 635                 ret = -EINVAL;
 636                 goto free_cpudata1;
 637         }
 638
 639         policy->cpuinfo.transition_latency = AMD_PSTATE_TRANSITION_LATENCY;
 640         policy->transition_delay_us = AMD_PSTATE_TRANSITION_DELAY;
 641
 642         policy->min = min_freq;
 643         policy->max = max_freq;
 644
 645         policy->cpuinfo.min_freq = min_freq;
 646         policy->cpuinfo.max_freq = max_freq;
 647
 648         /* It will be updated by governor */
 649         policy->cur = policy->cpuinfo.min_freq;
 650
 651         if (boot_cpu_has(X86_FEATURE_CPPC))
 652                 policy->fast_switch_possible = true;
 653
 654         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[0],
 655                                    FREQ_QOS_MIN, policy->cpuinfo.min_freq);
 656         if (ret < 0) {
 657                 dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
 658                 goto free_cpudata1;
 659         }
 660
 661         ret = freq_qos_add_request(&policy->constraints, &cpudata->req[1],
 662                                    FREQ_QOS_MAX, policy->cpuinfo.max_freq);
 663         if (ret < 0) {
 664                 dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
 665                 goto free_cpudata2;
 666         }
 667
 668         /* Initial processor data capability frequencies */
 669         cpudata->max_freq = max_freq;
 670         cpudata->min_freq = min_freq;
 671         cpudata->nominal_freq = nominal_freq;
 672         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
 673
 674         policy->driver_data = cpudata;
 675
 676         amd_pstate_boost_init(cpudata);
 677         if (!current_pstate_driver->adjust_perf)
 678                 current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
 679
 680         return 0;
 681
 682 free_cpudata2:
 683         freq_qos_remove_request(&cpudata->req[0]);
 684 free_cpudata1:
 685         kfree(cpudata);
 686         return ret;
 687 }
 688
 689 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
 690 {
 691         struct amd_cpudata *cpudata = policy->driver_data;
 692
 693         freq_qos_remove_request(&cpudata->req[1]);
 694         freq_qos_remove_request(&cpudata->req[0]);
 695         kfree(cpudata);
 696
 697         return 0;
 698 }
 699
 700 static int amd_pstate_cpu_resume(struct cpufreq_policy *policy)
 701 {
 702         int ret;
 703
 704         ret = amd_pstate_enable(true);
 705         if (ret)
 706                 pr_err("failed to enable amd-pstate during resume, return %d\n", ret);
 707
 708         return ret;
 709 }
 710
 711 static int amd_pstate_cpu_suspend(struct cpufreq_policy *policy)
 712 {
 713         int ret;
 714
 715         ret = amd_pstate_enable(false);
 716         if (ret)
 717                 pr_err("failed to disable amd-pstate during suspend, return %d\n", ret);
 718
 719         return ret;
 720 }
 721
 722 /* Sysfs attributes */
 723
 724 /*
 725  * This frequency is to indicate the maximum hardware frequency.
 726  * If boost is not active but supported, the frequency will be larger than the
 727  * one in cpuinfo.
 728  */
 729 static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
 730                                         char *buf)
 731 {
 732         int max_freq;
 733         struct amd_cpudata *cpudata = policy->driver_data;
 734
 735         max_freq = amd_get_max_freq(cpudata);
 736         if (max_freq < 0)
 737                 return max_freq;
 738
 739         return sysfs_emit(buf, "%u\n", max_freq);
 740 }
 741
 742 static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *policy,
 743                                                      char *buf)
 744 {
 745         int freq;
 746         struct amd_cpudata *cpudata = policy->driver_data;
 747
 748         freq = amd_get_lowest_nonlinear_freq(cpudata);
 749         if (freq < 0)
 750                 return freq;
 751
 752         return sysfs_emit(buf, "%u\n", freq);
 753 }
 754
 755 /*
 756  * In some of ASICs, the highest_perf is not the one in the _CPC table, so we
 757  * need to expose it to sysfs.
 758  */
 759 static ssize_t show_amd_pstate_highest_perf(struct cpufreq_policy *policy,
 760                                             char *buf)
 761 {
 762         u32 perf;
 763         struct amd_cpudata *cpudata = policy->driver_data;
 764
 765         perf = READ_ONCE(cpudata->highest_perf);
 766
 767         return sysfs_emit(buf, "%u\n", perf);
 768 }
 769
 770 static ssize_t show_energy_performance_available_preferences(
 771                                 struct cpufreq_policy *policy, char *buf)
 772 {
 773         int i = 0;
 774         int offset = 0;
 775
 776         while (energy_perf_strings[i] != NULL)
 777                 offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]);
 778
 779         sysfs_emit_at(buf, offset, "\n");
 780
 781         return offset;
 782 }
 783
 784 static ssize_t store_energy_performance_preference(
 785                 struct cpufreq_policy *policy, const char *buf, size_t count)
 786 {
 787         struct amd_cpudata *cpudata = policy->driver_data;
 788         char str_preference[21];
 789         ssize_t ret;
 790
 791         ret = sscanf(buf, "%20s", str_preference);
 792         if (ret != 1)
 793                 return -EINVAL;
 794
 795         ret = match_string(energy_perf_strings, -1, str_preference);
 796         if (ret < 0)
 797                 return -EINVAL;
 798
 799         mutex_lock(&amd_pstate_limits_lock);
 800         ret = amd_pstate_set_energy_pref_index(cpudata, ret);
 801         mutex_unlock(&amd_pstate_limits_lock);
 802
 803         return ret ?: count;
 804 }
 805
 806 static ssize_t show_energy_performance_preference(
 807                                 struct cpufreq_policy *policy, char *buf)
 808 {
 809         struct amd_cpudata *cpudata = policy->driver_data;
 810         int preference;
 811
 812         preference = amd_pstate_get_energy_pref_index(cpudata);
 813         if (preference < 0)
 814                 return preference;
 815
 816         return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]);
 817 }
 818
 819 static ssize_t amd_pstate_show_status(char *buf)
 820 {
 821         if (!current_pstate_driver)
 822                 return sysfs_emit(buf, "disable\n");
 823
 824         return sysfs_emit(buf, "%s\n", amd_pstate_mode_string[cppc_state]);
 825 }
 826
 827 static void amd_pstate_driver_cleanup(void)
 828 {
 829         current_pstate_driver = NULL;
 830 }
 831
 832 static int amd_pstate_update_status(const char *buf, size_t size)
 833 {
 834         int ret = 0;
 835         int mode_idx;
 836
 837         if (size > 7 || size < 6)
 838                 return -EINVAL;
 839         mode_idx = get_mode_idx_from_str(buf, size);
 840
 841         switch(mode_idx) {
 842         case AMD_PSTATE_DISABLE:
 843                 if (current_pstate_driver) {
 844                         cpufreq_unregister_driver(current_pstate_driver);
 845                         amd_pstate_driver_cleanup();
 846                 }
 847                 break;
 848         case AMD_PSTATE_PASSIVE:
 849                 if (current_pstate_driver) {
 850                         if (current_pstate_driver == &amd_pstate_driver)
 851                                 return 0;
 852                         cpufreq_unregister_driver(current_pstate_driver);
 853                 }
 854
 855                 current_pstate_driver = &amd_pstate_driver;
 856                 cppc_state = AMD_PSTATE_PASSIVE;
 857                 ret = cpufreq_register_driver(current_pstate_driver);
 858                 break;
 859         case AMD_PSTATE_ACTIVE:
 860                 if (current_pstate_driver) {
 861                         if (current_pstate_driver == &amd_pstate_epp_driver)
 862                                 return 0;
 863                         cpufreq_unregister_driver(current_pstate_driver);
 864                 }
 865
 866                 current_pstate_driver = &amd_pstate_epp_driver;
 867                 cppc_state = AMD_PSTATE_ACTIVE;
 868                 ret = cpufreq_register_driver(current_pstate_driver);
 869                 break;
 870         default:
 871                 ret = -EINVAL;
 872                 break;
 873         }
 874
 875         return ret;
 876 }
 877
 878 static ssize_t show_status(struct kobject *kobj,
 879                            struct kobj_attribute *attr, char *buf)
 880 {
 881         ssize_t ret;
 882
 883         mutex_lock(&amd_pstate_driver_lock);
 884         ret = amd_pstate_show_status(buf);
 885         mutex_unlock(&amd_pstate_driver_lock);
 886
 887         return ret;
 888 }
 889
 890 static ssize_t store_status(struct kobject *a, struct kobj_attribute *b,
 891                             const char *buf, size_t count)
 892 {
 893         char *p = memchr(buf, '\n', count);
 894         int ret;
 895
 896         mutex_lock(&amd_pstate_driver_lock);
 897         ret = amd_pstate_update_status(buf, p ? p - buf : count);
 898         mutex_unlock(&amd_pstate_driver_lock);
 899
 900         return ret < 0 ? ret : count;
 901 }
 902
 903 cpufreq_freq_attr_ro(amd_pstate_max_freq);
 904 cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq);
 905
 906 cpufreq_freq_attr_ro(amd_pstate_highest_perf);
 907 cpufreq_freq_attr_rw(energy_performance_preference);
 908 cpufreq_freq_attr_ro(energy_performance_available_preferences);
 909 define_one_global_rw(status);
 910
 911 static struct freq_attr *amd_pstate_attr[] = {
 912         &amd_pstate_max_freq,
 913         &amd_pstate_lowest_nonlinear_freq,
 914         &amd_pstate_highest_perf,
 915         NULL,
 916 };
 917
 918 static struct freq_attr *amd_pstate_epp_attr[] = {
 919         &amd_pstate_max_freq,
 920         &amd_pstate_lowest_nonlinear_freq,
 921         &amd_pstate_highest_perf,
 922         &energy_performance_preference,
 923         &energy_performance_available_preferences,
 924         NULL,
 925 };
 926
 927 static struct attribute *pstate_global_attributes[] = {
 928         &status.attr,
 929         NULL
 930 };
 931
 932 static const struct attribute_group amd_pstate_global_attr_group = {
 933         .attrs = pstate_global_attributes,
 934 };
 935
 936 static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
 937 {
 938         int min_freq, max_freq, nominal_freq, lowest_nonlinear_freq, ret;
 939         struct amd_cpudata *cpudata;
 940         struct device *dev;
 941         u64 value;
 942
 943         /*
 944          * Resetting PERF_CTL_MSR will put the CPU in P0 frequency,
 945          * which is ideal for initialization process.
 946          */
 947         amd_perf_ctl_reset(policy->cpu);
 948         dev = get_cpu_device(policy->cpu);
 949         if (!dev)
 950                 return -ENODEV;
 951
 952         cpudata = kzalloc(sizeof(*cpudata), GFP_KERNEL);
 953         if (!cpudata)
 954                 return -ENOMEM;
 955
 956         cpudata->cpu = policy->cpu;
 957         cpudata->epp_policy = 0;
 958
 959         ret = amd_pstate_init_perf(cpudata);
 960         if (ret)
 961                 goto free_cpudata1;
 962
 963         min_freq = amd_get_min_freq(cpudata);
 964         max_freq = amd_get_max_freq(cpudata);
 965         nominal_freq = amd_get_nominal_freq(cpudata);
 966         lowest_nonlinear_freq = amd_get_lowest_nonlinear_freq(cpudata);
 967         if (min_freq < 0 || max_freq < 0 || min_freq > max_freq) {
 968                 dev_err(dev, "min_freq(%d) or max_freq(%d) value is incorrect\n",
 969                                 min_freq, max_freq);
 970                 ret = -EINVAL;
 971                 goto free_cpudata1;
 972         }
 973
 974         policy->cpuinfo.min_freq = min_freq;
 975         policy->cpuinfo.max_freq = max_freq;
 976         /* It will be updated by governor */
 977         policy->cur = policy->cpuinfo.min_freq;
 978
 979         /* Initial processor data capability frequencies */
 980         cpudata->max_freq = max_freq;
 981         cpudata->min_freq = min_freq;
 982         cpudata->nominal_freq = nominal_freq;
 983         cpudata->lowest_nonlinear_freq = lowest_nonlinear_freq;
 984
 985         policy->driver_data = cpudata;
 986
 987         cpudata->epp_cached = amd_pstate_get_epp(cpudata, 0);
 988
 989         policy->min = policy->cpuinfo.min_freq;
 990         policy->max = policy->cpuinfo.max_freq;
 991
 992         /*
 993          * Set the policy to powersave to provide a valid fallback value in case
 994          * the default cpufreq governor is neither powersave nor performance.
 995          */
 996         policy->policy = CPUFREQ_POLICY_POWERSAVE;
 997
 998         if (boot_cpu_has(X86_FEATURE_CPPC)) {
 999                 policy->fast_switch_possible = true;
1000                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, &value);
1001                 if (ret)
1002                         return ret;
1003                 WRITE_ONCE(cpudata->cppc_req_cached, value);
1004
1005                 ret = rdmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1, &value);
1006                 if (ret)
1007                         return ret;
1008                 WRITE_ONCE(cpudata->cppc_cap1_cached, value);
1009         }
1010         amd_pstate_boost_init(cpudata);
1011
1012         return 0;
1013
1014 free_cpudata1:
1015         kfree(cpudata);
1016         return ret;
1017 }
1018
1019 static int amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
1020 {
1021         pr_debug("CPU %d exiting\n", policy->cpu);
1022         policy->fast_switch_possible = false;
1023         return 0;
1024 }
1025
1026 static void amd_pstate_epp_init(unsigned int cpu)
1027 {
1028         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1029         struct amd_cpudata *cpudata = policy->driver_data;
1030         u32 max_perf, min_perf;
1031         u64 value;
1032         s16 epp;
1033
1034         max_perf = READ_ONCE(cpudata->highest_perf);
1035         min_perf = READ_ONCE(cpudata->lowest_perf);
1036
1037         value = READ_ONCE(cpudata->cppc_req_cached);
1038
1039         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1040                 min_perf = max_perf;
1041
1042         /* Initial min/max values for CPPC Performance Controls Register */
1043         value &= ~AMD_CPPC_MIN_PERF(~0L);
1044         value |= AMD_CPPC_MIN_PERF(min_perf);
1045
1046         value &= ~AMD_CPPC_MAX_PERF(~0L);
1047         value |= AMD_CPPC_MAX_PERF(max_perf);
1048
1049         /* CPPC EPP feature require to set zero to the desire perf bit */
1050         value &= ~AMD_CPPC_DES_PERF(~0L);
1051         value |= AMD_CPPC_DES_PERF(0);
1052
1053         if (cpudata->epp_policy == cpudata->policy)
1054                 goto skip_epp;
1055
1056         cpudata->epp_policy = cpudata->policy;
1057
1058         /* Get BIOS pre-defined epp value */
1059         epp = amd_pstate_get_epp(cpudata, value);
1060         if (epp < 0) {
1061                 /**
1062                  * This return value can only be negative for shared_memory
1063                  * systems where EPP register read/write not supported.
1064                  */
1065                 goto skip_epp;
1066         }
1067
1068         if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE)
1069                 epp = 0;
1070
1071         /* Set initial EPP value */
1072         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1073                 value &= ~GENMASK_ULL(31, 24);
1074                 value |= (u64)epp << 24;
1075         }
1076
1077         WRITE_ONCE(cpudata->cppc_req_cached, value);
1078         amd_pstate_set_epp(cpudata, epp);
1079 skip_epp:
1080         cpufreq_cpu_put(policy);
1081 }
1082
1083 static int amd_pstate_epp_set_policy(struct cpufreq_policy *policy)
1084 {
1085         struct amd_cpudata *cpudata = policy->driver_data;
1086
1087         if (!policy->cpuinfo.max_freq)
1088                 return -ENODEV;
1089
1090         pr_debug("set_policy: cpuinfo.max %u policy->max %u\n",
1091                                 policy->cpuinfo.max_freq, policy->max);
1092
1093         cpudata->policy = policy->policy;
1094
1095         amd_pstate_epp_init(policy->cpu);
1096
1097         return 0;
1098 }
1099
1100 static void amd_pstate_epp_reenable(struct amd_cpudata *cpudata)
1101 {
1102         struct cppc_perf_ctrls perf_ctrls;
1103         u64 value, max_perf;
1104         int ret;
1105
1106         ret = amd_pstate_enable(true);
1107         if (ret)
1108                 pr_err("failed to enable amd pstate during resume, return %d\n", ret);
1109
1110         value = READ_ONCE(cpudata->cppc_req_cached);
1111         max_perf = READ_ONCE(cpudata->highest_perf);
1112
1113         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1114                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1115         } else {
1116                 perf_ctrls.max_perf = max_perf;
1117                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(cpudata->epp_cached);
1118                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1119         }
1120 }
1121
1122 static int amd_pstate_epp_cpu_online(struct cpufreq_policy *policy)
1123 {
1124         struct amd_cpudata *cpudata = policy->driver_data;
1125
1126         pr_debug("AMD CPU Core %d going online\n", cpudata->cpu);
1127
1128         if (cppc_state == AMD_PSTATE_ACTIVE) {
1129                 amd_pstate_epp_reenable(cpudata);
1130                 cpudata->suspended = false;
1131         }
1132
1133         return 0;
1134 }
1135
1136 static void amd_pstate_epp_offline(struct cpufreq_policy *policy)
1137 {
1138         struct amd_cpudata *cpudata = policy->driver_data;
1139         struct cppc_perf_ctrls perf_ctrls;
1140         int min_perf;
1141         u64 value;
1142
1143         min_perf = READ_ONCE(cpudata->lowest_perf);
1144         value = READ_ONCE(cpudata->cppc_req_cached);
1145
1146         mutex_lock(&amd_pstate_limits_lock);
1147         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1148                 cpudata->epp_policy = CPUFREQ_POLICY_UNKNOWN;
1149
1150                 /* Set max perf same as min perf */
1151                 value &= ~AMD_CPPC_MAX_PERF(~0L);
1152                 value |= AMD_CPPC_MAX_PERF(min_perf);
1153                 value &= ~AMD_CPPC_MIN_PERF(~0L);
1154                 value |= AMD_CPPC_MIN_PERF(min_perf);
1155                 wrmsrl_on_cpu(cpudata->cpu, MSR_AMD_CPPC_REQ, value);
1156         } else {
1157                 perf_ctrls.desired_perf = 0;
1158                 perf_ctrls.max_perf = min_perf;
1159                 perf_ctrls.energy_perf = AMD_CPPC_ENERGY_PERF_PREF(HWP_EPP_BALANCE_POWERSAVE);
1160                 cppc_set_perf(cpudata->cpu, &perf_ctrls);
1161         }
1162         mutex_unlock(&amd_pstate_limits_lock);
1163 }
1164
1165 static int amd_pstate_epp_cpu_offline(struct cpufreq_policy *policy)
1166 {
1167         struct amd_cpudata *cpudata = policy->driver_data;
1168
1169         pr_debug("AMD CPU Core %d going offline\n", cpudata->cpu);
1170
1171         if (cpudata->suspended)
1172                 return 0;
1173
1174         if (cppc_state == AMD_PSTATE_ACTIVE)
1175                 amd_pstate_epp_offline(policy);
1176
1177         return 0;
1178 }
1179
1180 static int amd_pstate_epp_verify_policy(struct cpufreq_policy_data *policy)
1181 {
1182         cpufreq_verify_within_cpu_limits(policy);
1183         pr_debug("policy_max =%d, policy_min=%d\n", policy->max, policy->min);
1184         return 0;
1185 }
1186
1187 static int amd_pstate_epp_suspend(struct cpufreq_policy *policy)
1188 {
1189         struct amd_cpudata *cpudata = policy->driver_data;
1190         int ret;
1191
1192         /* avoid suspending when EPP is not enabled */
1193         if (cppc_state != AMD_PSTATE_ACTIVE)
1194                 return 0;
1195
1196         /* set this flag to avoid setting core offline*/
1197         cpudata->suspended = true;
1198
1199         /* disable CPPC in lowlevel firmware */
1200         ret = amd_pstate_enable(false);
1201         if (ret)
1202                 pr_err("failed to suspend, return %d\n", ret);
1203
1204         return 0;
1205 }
1206
1207 static int amd_pstate_epp_resume(struct cpufreq_policy *policy)
1208 {
1209         struct amd_cpudata *cpudata = policy->driver_data;
1210
1211         if (cpudata->suspended) {
1212                 mutex_lock(&amd_pstate_limits_lock);
1213
1214                 /* enable amd pstate from suspend state*/
1215                 amd_pstate_epp_reenable(cpudata);
1216
1217                 mutex_unlock(&amd_pstate_limits_lock);
1218
1219                 cpudata->suspended = false;
1220         }
1221
1222         return 0;
1223 }
1224
1225 static struct cpufreq_driver amd_pstate_driver = {
1226         .flags          = CPUFREQ_CONST_LOOPS | CPUFREQ_NEED_UPDATE_LIMITS,
1227         .verify         = amd_pstate_verify,
1228         .target         = amd_pstate_target,
1229         .init           = amd_pstate_cpu_init,
1230         .exit           = amd_pstate_cpu_exit,
1231         .suspend        = amd_pstate_cpu_suspend,
1232         .resume         = amd_pstate_cpu_resume,
1233         .set_boost      = amd_pstate_set_boost,
1234         .name           = "amd-pstate",
1235         .attr           = amd_pstate_attr,
1236 };
1237
1238 static struct cpufreq_driver amd_pstate_epp_driver = {
1239         .flags          = CPUFREQ_CONST_LOOPS,
1240         .verify         = amd_pstate_epp_verify_policy,
1241         .setpolicy      = amd_pstate_epp_set_policy,
1242         .init           = amd_pstate_epp_cpu_init,
1243         .exit           = amd_pstate_epp_cpu_exit,
1244         .offline        = amd_pstate_epp_cpu_offline,
1245         .online         = amd_pstate_epp_cpu_online,
1246         .suspend        = amd_pstate_epp_suspend,
1247         .resume         = amd_pstate_epp_resume,
1248         .name           = "amd_pstate_epp",
1249         .attr           = amd_pstate_epp_attr,
1250 };
1251
1252 static int __init amd_pstate_init(void)
1253 {
1254         int ret;
1255
1256         if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
1257                 return -ENODEV;
1258         /*
1259          * by default the pstate driver is disabled to load
1260          * enable the amd_pstate passive mode driver explicitly
1261          * with amd_pstate=passive or other modes in kernel command line
1262          */
1263         if (cppc_state == AMD_PSTATE_DISABLE) {
1264                 pr_info("driver load is disabled, boot with specific mode to enable this\n");
1265                 return -ENODEV;
1266         }
1267
1268         if (!acpi_cpc_valid()) {
1269                 pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
1270                 return -ENODEV;
1271         }
1272
1273         /* don't keep reloading if cpufreq_driver exists */
1274         if (cpufreq_get_current_driver())
1275                 return -EEXIST;
1276
1277         /* capability check */
1278         if (boot_cpu_has(X86_FEATURE_CPPC)) {
1279                 pr_debug("AMD CPPC MSR based functionality is supported\n");
1280                 if (cppc_state == AMD_PSTATE_PASSIVE)
1281                         current_pstate_driver->adjust_perf = amd_pstate_adjust_perf;
1282         } else {
1283                 pr_debug("AMD CPPC shared memory based functionality is supported\n");
1284                 static_call_update(amd_pstate_enable, cppc_enable);
1285                 static_call_update(amd_pstate_init_perf, cppc_init_perf);
1286                 static_call_update(amd_pstate_update_perf, cppc_update_perf);
1287         }
1288
1289         /* enable amd pstate feature */
1290         ret = amd_pstate_enable(true);
1291         if (ret) {
1292                 pr_err("failed to enable with return %d\n", ret);
1293                 return ret;
1294         }
1295
1296         ret = cpufreq_register_driver(current_pstate_driver);
1297         if (ret)
1298                 pr_err("failed to register with return %d\n", ret);
1299
1300         amd_pstate_kobj = kobject_create_and_add("amd_pstate", &cpu_subsys.dev_root->kobj);
1301         if (!amd_pstate_kobj) {
1302                 ret = -EINVAL;
1303                 pr_err("global sysfs registration failed.\n");
1304                 goto kobject_free;
1305         }
1306
1307         ret = sysfs_create_group(amd_pstate_kobj, &amd_pstate_global_attr_group);
1308         if (ret) {
1309                 pr_err("sysfs attribute export failed with error %d.\n", ret);
1310                 goto global_attr_free;
1311         }
1312
1313         return ret;
1314
1315 global_attr_free:
1316         kobject_put(amd_pstate_kobj);
1317 kobject_free:
1318         cpufreq_unregister_driver(current_pstate_driver);
1319         return ret;
1320 }
1321 device_initcall(amd_pstate_init);
1322
1323 static int __init amd_pstate_param(char *str)
1324 {
1325         size_t size;
1326         int mode_idx;
1327
1328         if (!str)
1329                 return -EINVAL;
1330
1331         size = strlen(str);
1332         mode_idx = get_mode_idx_from_str(str, size);
1333
1334         if (mode_idx >= AMD_PSTATE_DISABLE && mode_idx < AMD_PSTATE_MAX) {
1335                 cppc_state = mode_idx;
1336                 if (cppc_state == AMD_PSTATE_DISABLE)
1337                         pr_info("driver is explicitly disabled\n");
1338
1339                 if (cppc_state == AMD_PSTATE_ACTIVE)
1340                         current_pstate_driver = &amd_pstate_epp_driver;
1341
1342                 if (cppc_state == AMD_PSTATE_PASSIVE)
1343                         current_pstate_driver = &amd_pstate_driver;
1344
1345                 return 0;
1346         }
1347
1348         return -EINVAL;
1349 }
1350 early_param("amd_pstate", amd_pstate_param);
1351
1352 MODULE_AUTHOR("Huang Rui <ray.huang@amd.com>");
1353 MODULE_DESCRIPTION("AMD Processor P-state Frequency Driver");