drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 static LIST_HEAD(cpufreq_policy_list);
  35
  36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  37 {
  38         return cpumask_empty(policy->cpus);
  39 }
  40
  41 /* Macros to iterate over CPU policies */
  42 #define for_each_suitable_policy(__policy, __active)                     \
  43         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
  44                 if ((__active) == !policy_is_inactive(__policy))
  45
  46 #define for_each_active_policy(__policy)                \
  47         for_each_suitable_policy(__policy, true)
  48 #define for_each_inactive_policy(__policy)              \
  49         for_each_suitable_policy(__policy, false)
  50
  51 #define for_each_policy(__policy)                       \
  52         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  53
  54 /* Iterate over governors */
  55 static LIST_HEAD(cpufreq_governor_list);
  56 #define for_each_governor(__governor)                           \
  57         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  58
  59 /**
  60  * The "cpufreq driver" - the arch- or hardware-dependent low
  61  * level driver of CPUFreq support, and its spinlock. This lock
  62  * also protects the cpufreq_cpu_data array.
  63  */
  64 static struct cpufreq_driver *cpufreq_driver;
  65 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  66 static DEFINE_RWLOCK(cpufreq_driver_lock);
  67
  68 /* Flag to suspend/resume CPUFreq governors */
  69 static bool cpufreq_suspended;
  70
  71 static inline bool has_target(void)
  72 {
  73         return cpufreq_driver->target_index || cpufreq_driver->target;
  74 }
  75
  76 /* internal prototypes */
  77 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
  78 static int cpufreq_init_governor(struct cpufreq_policy *policy);
  79 static void cpufreq_exit_governor(struct cpufreq_policy *policy);
  80 static int cpufreq_start_governor(struct cpufreq_policy *policy);
  81 static void cpufreq_stop_governor(struct cpufreq_policy *policy);
  82 static void cpufreq_governor_limits(struct cpufreq_policy *policy);
  83
  84 /**
  85  * Two notifier lists: the "policy" list is involved in the
  86  * validation process for a new CPU frequency policy; the
  87  * "transition" list for kernel code that needs to handle
  88  * changes to devices when the CPU clock speed changes.
  89  * The mutex locks both lists.
  90  */
  91 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  92 SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
  93
  94 static int off __read_mostly;
  95 static int cpufreq_disabled(void)
  96 {
  97         return off;
  98 }
  99 void disable_cpufreq(void)
 100 {
 101         off = 1;
 102 }
 103 static DEFINE_MUTEX(cpufreq_governor_mutex);
 104
 105 bool have_governor_per_policy(void)
 106 {
 107         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 108 }
 109 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 110
 111 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 112 {
 113         if (have_governor_per_policy())
 114                 return &policy->kobj;
 115         else
 116                 return cpufreq_global_kobject;
 117 }
 118 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 119
 120 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 121 {
 122         u64 idle_time;
 123         u64 cur_wall_time;
 124         u64 busy_time;
 125
 126         cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
 127
 128         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 129         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 130         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 131         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 132         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 133         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 134
 135         idle_time = cur_wall_time - busy_time;
 136         if (wall)
 137                 *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
 138
 139         return div_u64(idle_time, NSEC_PER_USEC);
 140 }
 141
 142 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 143 {
 144         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 145
 146         if (idle_time == -1ULL)
 147                 return get_cpu_idle_time_jiffy(cpu, wall);
 148         else if (!io_busy)
 149                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 150
 151         return idle_time;
 152 }
 153 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 154
 155 __weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
 156                 unsigned long max_freq)
 157 {
 158 }
 159 EXPORT_SYMBOL_GPL(arch_set_freq_scale);
 160
 161 /*
 162  * This is a generic cpufreq init() routine which can be used by cpufreq
 163  * drivers of SMP systems. It will do following:
 164  * - validate & show freq table passed
 165  * - set policies transition latency
 166  * - policy->cpus with all possible CPUs
 167  */
 168 int cpufreq_generic_init(struct cpufreq_policy *policy,
 169                 struct cpufreq_frequency_table *table,
 170                 unsigned int transition_latency)
 171 {
 172         policy->freq_table = table;
 173         policy->cpuinfo.transition_latency = transition_latency;
 174
 175         /*
 176          * The driver only supports the SMP configuration where all processors
 177          * share the clock and voltage and clock.
 178          */
 179         cpumask_setall(policy->cpus);
 180
 181         return 0;
 182 }
 183 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 184
 185 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 186 {
 187         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 188
 189         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 190 }
 191 EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
 192
 193 unsigned int cpufreq_generic_get(unsigned int cpu)
 194 {
 195         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 196
 197         if (!policy || IS_ERR(policy->clk)) {
 198                 pr_err("%s: No %s associated to cpu: %d\n",
 199                        __func__, policy ? "clk" : "policy", cpu);
 200                 return 0;
 201         }
 202
 203         return clk_get_rate(policy->clk) / 1000;
 204 }
 205 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 206
 207 /**
 208  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 209  *
 210  * @cpu: cpu to find policy for.
 211  *
 212  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 213  * It also increments the kobject reference count to mark it busy and so would
 214  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 215  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 216  * freed as that depends on the kobj count.
 217  *
 218  * Return: A valid policy on success, otherwise NULL on failure.
 219  */
 220 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 221 {
 222         struct cpufreq_policy *policy = NULL;
 223         unsigned long flags;
 224
 225         if (WARN_ON(cpu >= nr_cpu_ids))
 226                 return NULL;
 227
 228         /* get the cpufreq driver */
 229         read_lock_irqsave(&cpufreq_driver_lock, flags);
 230
 231         if (cpufreq_driver) {
 232                 /* get the CPU */
 233                 policy = cpufreq_cpu_get_raw(cpu);
 234                 if (policy)
 235                         kobject_get(&policy->kobj);
 236         }
 237
 238         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 239
 240         return policy;
 241 }
 242 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 243
 244 /**
 245  * cpufreq_cpu_put: Decrements the usage count of a policy
 246  *
 247  * @policy: policy earlier returned by cpufreq_cpu_get().
 248  *
 249  * This decrements the kobject reference count incremented earlier by calling
 250  * cpufreq_cpu_get().
 251  */
 252 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 253 {
 254         kobject_put(&policy->kobj);
 255 }
 256 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 257
 258 /*********************************************************************
 259  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 260  *********************************************************************/
 261
 262 /**
 263  * adjust_jiffies - adjust the system "loops_per_jiffy"
 264  *
 265  * This function alters the system "loops_per_jiffy" for the clock
 266  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 267  * systems as each CPU might be scaled differently. So, use the arch
 268  * per-CPU loops_per_jiffy value wherever possible.
 269  */
 270 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 271 {
 272 #ifndef CONFIG_SMP
 273         static unsigned long l_p_j_ref;
 274         static unsigned int l_p_j_ref_freq;
 275
 276         if (ci->flags & CPUFREQ_CONST_LOOPS)
 277                 return;
 278
 279         if (!l_p_j_ref_freq) {
 280                 l_p_j_ref = loops_per_jiffy;
 281                 l_p_j_ref_freq = ci->old;
 282                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 283                          l_p_j_ref, l_p_j_ref_freq);
 284         }
 285         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 286                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 287                                                                 ci->new);
 288                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 289                          loops_per_jiffy, ci->new);
 290         }
 291 #endif
 292 }
 293
 294 /**
 295  * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies.
 296  * @policy: cpufreq policy to enable fast frequency switching for.
 297  * @freqs: contain details of the frequency update.
 298  * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
 299  *
 300  * This function calls the transition notifiers and the "adjust_jiffies"
 301  * function. It is called twice on all CPU frequency changes that have
 302  * external effects.
 303  */
 304 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 305                                       struct cpufreq_freqs *freqs,
 306                                       unsigned int state)
 307 {
 308         BUG_ON(irqs_disabled());
 309
 310         if (cpufreq_disabled())
 311                 return;
 312
 313         freqs->flags = cpufreq_driver->flags;
 314         pr_debug("notification %u of frequency transition to %u kHz\n",
 315                  state, freqs->new);
 316
 317         switch (state) {
 318         case CPUFREQ_PRECHANGE:
 319                 /*
 320                  * Detect if the driver reported a value as "old frequency"
 321                  * which is not equal to what the cpufreq core thinks is
 322                  * "old frequency".
 323                  */
 324                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 325                         if (policy->cur && (policy->cur != freqs->old)) {
 326                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 327                                          freqs->old, policy->cur);
 328                                 freqs->old = policy->cur;
 329                         }
 330                 }
 331
 332                 for_each_cpu(freqs->cpu, policy->cpus) {
 333                         srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 334                                                  CPUFREQ_PRECHANGE, freqs);
 335                 }
 336
 337                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 338                 break;
 339
 340         case CPUFREQ_POSTCHANGE:
 341                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 342                 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
 343                          cpumask_pr_args(policy->cpus));
 344
 345                 for_each_cpu(freqs->cpu, policy->cpus) {
 346                         trace_cpu_frequency(freqs->new, freqs->cpu);
 347                         srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 348                                                  CPUFREQ_POSTCHANGE, freqs);
 349                 }
 350
 351                 cpufreq_stats_record_transition(policy, freqs->new);
 352                 policy->cur = freqs->new;
 353         }
 354 }
 355
 356 /* Do post notifications when there are chances that transition has failed */
 357 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 358                 struct cpufreq_freqs *freqs, int transition_failed)
 359 {
 360         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 361         if (!transition_failed)
 362                 return;
 363
 364         swap(freqs->old, freqs->new);
 365         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 366         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 367 }
 368
 369 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 370                 struct cpufreq_freqs *freqs)
 371 {
 372
 373         /*
 374          * Catch double invocations of _begin() which lead to self-deadlock.
 375          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 376          * doesn't invoke _begin() on their behalf, and hence the chances of
 377          * double invocations are very low. Moreover, there are scenarios
 378          * where these checks can emit false-positive warnings in these
 379          * drivers; so we avoid that by skipping them altogether.
 380          */
 381         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 382                                 && current == policy->transition_task);
 383
 384 wait:
 385         wait_event(policy->transition_wait, !policy->transition_ongoing);
 386
 387         spin_lock(&policy->transition_lock);
 388
 389         if (unlikely(policy->transition_ongoing)) {
 390                 spin_unlock(&policy->transition_lock);
 391                 goto wait;
 392         }
 393
 394         policy->transition_ongoing = true;
 395         policy->transition_task = current;
 396
 397         spin_unlock(&policy->transition_lock);
 398
 399         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 400 }
 401 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 402
 403 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 404                 struct cpufreq_freqs *freqs, int transition_failed)
 405 {
 406         if (WARN_ON(!policy->transition_ongoing))
 407                 return;
 408
 409         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 410
 411         policy->transition_ongoing = false;
 412         policy->transition_task = NULL;
 413
 414         wake_up(&policy->transition_wait);
 415 }
 416 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 417
 418 /*
 419  * Fast frequency switching status count.  Positive means "enabled", negative
 420  * means "disabled" and 0 means "not decided yet".
 421  */
 422 static int cpufreq_fast_switch_count;
 423 static DEFINE_MUTEX(cpufreq_fast_switch_lock);
 424
 425 static void cpufreq_list_transition_notifiers(void)
 426 {
 427         struct notifier_block *nb;
 428
 429         pr_info("Registered transition notifiers:\n");
 430
 431         mutex_lock(&cpufreq_transition_notifier_list.mutex);
 432
 433         for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
 434                 pr_info("%pF\n", nb->notifier_call);
 435
 436         mutex_unlock(&cpufreq_transition_notifier_list.mutex);
 437 }
 438
 439 /**
 440  * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
 441  * @policy: cpufreq policy to enable fast frequency switching for.
 442  *
 443  * Try to enable fast frequency switching for @policy.
 444  *
 445  * The attempt will fail if there is at least one transition notifier registered
 446  * at this point, as fast frequency switching is quite fundamentally at odds
 447  * with transition notifiers.  Thus if successful, it will make registration of
 448  * transition notifiers fail going forward.
 449  */
 450 void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
 451 {
 452         lockdep_assert_held(&policy->rwsem);
 453
 454         if (!policy->fast_switch_possible)
 455                 return;
 456
 457         mutex_lock(&cpufreq_fast_switch_lock);
 458         if (cpufreq_fast_switch_count >= 0) {
 459                 cpufreq_fast_switch_count++;
 460                 policy->fast_switch_enabled = true;
 461         } else {
 462                 pr_warn("CPU%u: Fast frequency switching not enabled\n",
 463                         policy->cpu);
 464                 cpufreq_list_transition_notifiers();
 465         }
 466         mutex_unlock(&cpufreq_fast_switch_lock);
 467 }
 468 EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
 469
 470 /**
 471  * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
 472  * @policy: cpufreq policy to disable fast frequency switching for.
 473  */
 474 void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
 475 {
 476         mutex_lock(&cpufreq_fast_switch_lock);
 477         if (policy->fast_switch_enabled) {
 478                 policy->fast_switch_enabled = false;
 479                 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
 480                         cpufreq_fast_switch_count--;
 481         }
 482         mutex_unlock(&cpufreq_fast_switch_lock);
 483 }
 484 EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
 485
 486 /**
 487  * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
 488  * one.
 489  * @target_freq: target frequency to resolve.
 490  *
 491  * The target to driver frequency mapping is cached in the policy.
 492  *
 493  * Return: Lowest driver-supported frequency greater than or equal to the
 494  * given target_freq, subject to policy (min/max) and driver limitations.
 495  */
 496 unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
 497                                          unsigned int target_freq)
 498 {
 499         target_freq = clamp_val(target_freq, policy->min, policy->max);
 500         policy->cached_target_freq = target_freq;
 501
 502         if (cpufreq_driver->target_index) {
 503                 int idx;
 504
 505                 idx = cpufreq_frequency_table_target(policy, target_freq,
 506                                                      CPUFREQ_RELATION_L);
 507                 policy->cached_resolved_idx = idx;
 508                 return policy->freq_table[idx].frequency;
 509         }
 510
 511         if (cpufreq_driver->resolve_freq)
 512                 return cpufreq_driver->resolve_freq(policy, target_freq);
 513
 514         return target_freq;
 515 }
 516 EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
 517
 518 unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
 519 {
 520         unsigned int latency;
 521
 522         if (policy->transition_delay_us)
 523                 return policy->transition_delay_us;
 524
 525         latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
 526         if (latency) {
 527                 /*
 528                  * For platforms that can change the frequency very fast (< 10
 529                  * us), the above formula gives a decent transition delay. But
 530                  * for platforms where transition_latency is in milliseconds, it
 531                  * ends up giving unrealistic values.
 532                  *
 533                  * Cap the default transition delay to 10 ms, which seems to be
 534                  * a reasonable amount of time after which we should reevaluate
 535                  * the frequency.
 536                  */
 537                 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
 538         }
 539
 540         return LATENCY_MULTIPLIER;
 541 }
 542 EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
 543
 544 /*********************************************************************
 545  *                          SYSFS INTERFACE                          *
 546  *********************************************************************/
 547 static ssize_t show_boost(struct kobject *kobj,
 548                                  struct attribute *attr, char *buf)
 549 {
 550         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 551 }
 552
 553 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 554                                   const char *buf, size_t count)
 555 {
 556         int ret, enable;
 557
 558         ret = sscanf(buf, "%d", &enable);
 559         if (ret != 1 || enable < 0 || enable > 1)
 560                 return -EINVAL;
 561
 562         if (cpufreq_boost_trigger_state(enable)) {
 563                 pr_err("%s: Cannot %s BOOST!\n",
 564                        __func__, enable ? "enable" : "disable");
 565                 return -EINVAL;
 566         }
 567
 568         pr_debug("%s: cpufreq BOOST %s\n",
 569                  __func__, enable ? "enabled" : "disabled");
 570
 571         return count;
 572 }
 573 define_one_global_rw(boost);
 574
 575 static struct cpufreq_governor *find_governor(const char *str_governor)
 576 {
 577         struct cpufreq_governor *t;
 578
 579         for_each_governor(t)
 580                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 581                         return t;
 582
 583         return NULL;
 584 }
 585
 586 /**
 587  * cpufreq_parse_governor - parse a governor string
 588  */
 589 static int cpufreq_parse_governor(char *str_governor,
 590                                   struct cpufreq_policy *policy)
 591 {
 592         if (cpufreq_driver->setpolicy) {
 593                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 594                         policy->policy = CPUFREQ_POLICY_PERFORMANCE;
 595                         return 0;
 596                 }
 597
 598                 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) {
 599                         policy->policy = CPUFREQ_POLICY_POWERSAVE;
 600                         return 0;
 601                 }
 602         } else {
 603                 struct cpufreq_governor *t;
 604
 605                 mutex_lock(&cpufreq_governor_mutex);
 606
 607                 t = find_governor(str_governor);
 608                 if (!t) {
 609                         int ret;
 610
 611                         mutex_unlock(&cpufreq_governor_mutex);
 612
 613                         ret = request_module("cpufreq_%s", str_governor);
 614                         if (ret)
 615                                 return -EINVAL;
 616
 617                         mutex_lock(&cpufreq_governor_mutex);
 618
 619                         t = find_governor(str_governor);
 620                 }
 621                 if (t && !try_module_get(t->owner))
 622                         t = NULL;
 623
 624                 mutex_unlock(&cpufreq_governor_mutex);
 625
 626                 if (t) {
 627                         policy->governor = t;
 628                         return 0;
 629                 }
 630         }
 631
 632         return -EINVAL;
 633 }
 634
 635 /**
 636  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 637  * print out cpufreq information
 638  *
 639  * Write out information from cpufreq_driver->policy[cpu]; object must be
 640  * "unsigned int".
 641  */
 642
 643 #define show_one(file_name, object)                     \
 644 static ssize_t show_##file_name                         \
 645 (struct cpufreq_policy *policy, char *buf)              \
 646 {                                                       \
 647         return sprintf(buf, "%u\n", policy->object);    \
 648 }
 649
 650 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 651 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 652 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 653 show_one(scaling_min_freq, min);
 654 show_one(scaling_max_freq, max);
 655
 656 __weak unsigned int arch_freq_get_on_cpu(int cpu)
 657 {
 658         return 0;
 659 }
 660
 661 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 662 {
 663         ssize_t ret;
 664         unsigned int freq;
 665
 666         freq = arch_freq_get_on_cpu(policy->cpu);
 667         if (freq)
 668                 ret = sprintf(buf, "%u\n", freq);
 669         else if (cpufreq_driver && cpufreq_driver->setpolicy &&
 670                         cpufreq_driver->get)
 671                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 672         else
 673                 ret = sprintf(buf, "%u\n", policy->cur);
 674         return ret;
 675 }
 676
 677 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 678                                 struct cpufreq_policy *new_policy);
 679
 680 /**
 681  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 682  */
 683 #define store_one(file_name, object)                    \
 684 static ssize_t store_##file_name                                        \
 685 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 686 {                                                                       \
 687         int ret, temp;                                                  \
 688         struct cpufreq_policy new_policy;                               \
 689                                                                         \
 690         memcpy(&new_policy, policy, sizeof(*policy));                   \
 691         new_policy.min = policy->user_policy.min;                       \
 692         new_policy.max = policy->user_policy.max;                       \
 693                                                                         \
 694         ret = sscanf(buf, "%u", &new_policy.object);                    \
 695         if (ret != 1)                                                   \
 696                 return -EINVAL;                                         \
 697                                                                         \
 698         temp = new_policy.object;                                       \
 699         ret = cpufreq_set_policy(policy, &new_policy);          \
 700         if (!ret)                                                       \
 701                 policy->user_policy.object = temp;                      \
 702                                                                         \
 703         return ret ? ret : count;                                       \
 704 }
 705
 706 store_one(scaling_min_freq, min);
 707 store_one(scaling_max_freq, max);
 708
 709 /**
 710  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 711  */
 712 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 713                                         char *buf)
 714 {
 715         unsigned int cur_freq = __cpufreq_get(policy);
 716
 717         if (cur_freq)
 718                 return sprintf(buf, "%u\n", cur_freq);
 719
 720         return sprintf(buf, "<unknown>\n");
 721 }
 722
 723 /**
 724  * show_scaling_governor - show the current policy for the specified CPU
 725  */
 726 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 727 {
 728         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 729                 return sprintf(buf, "powersave\n");
 730         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 731                 return sprintf(buf, "performance\n");
 732         else if (policy->governor)
 733                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 734                                 policy->governor->name);
 735         return -EINVAL;
 736 }
 737
 738 /**
 739  * store_scaling_governor - store policy for the specified CPU
 740  */
 741 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 742                                         const char *buf, size_t count)
 743 {
 744         int ret;
 745         char    str_governor[16];
 746         struct cpufreq_policy new_policy;
 747
 748         memcpy(&new_policy, policy, sizeof(*policy));
 749
 750         ret = sscanf(buf, "%15s", str_governor);
 751         if (ret != 1)
 752                 return -EINVAL;
 753
 754         if (cpufreq_parse_governor(str_governor, &new_policy))
 755                 return -EINVAL;
 756
 757         ret = cpufreq_set_policy(policy, &new_policy);
 758
 759         if (new_policy.governor)
 760                 module_put(new_policy.governor->owner);
 761
 762         return ret ? ret : count;
 763 }
 764
 765 /**
 766  * show_scaling_driver - show the cpufreq driver currently loaded
 767  */
 768 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 769 {
 770         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 771 }
 772
 773 /**
 774  * show_scaling_available_governors - show the available CPUfreq governors
 775  */
 776 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 777                                                 char *buf)
 778 {
 779         ssize_t i = 0;
 780         struct cpufreq_governor *t;
 781
 782         if (!has_target()) {
 783                 i += sprintf(buf, "performance powersave");
 784                 goto out;
 785         }
 786
 787         for_each_governor(t) {
 788                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 789                     - (CPUFREQ_NAME_LEN + 2)))
 790                         goto out;
 791                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 792         }
 793 out:
 794         i += sprintf(&buf[i], "\n");
 795         return i;
 796 }
 797
 798 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 799 {
 800         ssize_t i = 0;
 801         unsigned int cpu;
 802
 803         for_each_cpu(cpu, mask) {
 804                 if (i)
 805                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 806                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 807                 if (i >= (PAGE_SIZE - 5))
 808                         break;
 809         }
 810         i += sprintf(&buf[i], "\n");
 811         return i;
 812 }
 813 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 814
 815 /**
 816  * show_related_cpus - show the CPUs affected by each transition even if
 817  * hw coordination is in use
 818  */
 819 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 820 {
 821         return cpufreq_show_cpus(policy->related_cpus, buf);
 822 }
 823
 824 /**
 825  * show_affected_cpus - show the CPUs affected by each transition
 826  */
 827 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 828 {
 829         return cpufreq_show_cpus(policy->cpus, buf);
 830 }
 831
 832 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 833                                         const char *buf, size_t count)
 834 {
 835         unsigned int freq = 0;
 836         unsigned int ret;
 837
 838         if (!policy->governor || !policy->governor->store_setspeed)
 839                 return -EINVAL;
 840
 841         ret = sscanf(buf, "%u", &freq);
 842         if (ret != 1)
 843                 return -EINVAL;
 844
 845         policy->governor->store_setspeed(policy, freq);
 846
 847         return count;
 848 }
 849
 850 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 851 {
 852         if (!policy->governor || !policy->governor->show_setspeed)
 853                 return sprintf(buf, "<unsupported>\n");
 854
 855         return policy->governor->show_setspeed(policy, buf);
 856 }
 857
 858 /**
 859  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 860  */
 861 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 862 {
 863         unsigned int limit;
 864         int ret;
 865         if (cpufreq_driver->bios_limit) {
 866                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 867                 if (!ret)
 868                         return sprintf(buf, "%u\n", limit);
 869         }
 870         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 871 }
 872
 873 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 874 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 875 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 876 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 877 cpufreq_freq_attr_ro(scaling_available_governors);
 878 cpufreq_freq_attr_ro(scaling_driver);
 879 cpufreq_freq_attr_ro(scaling_cur_freq);
 880 cpufreq_freq_attr_ro(bios_limit);
 881 cpufreq_freq_attr_ro(related_cpus);
 882 cpufreq_freq_attr_ro(affected_cpus);
 883 cpufreq_freq_attr_rw(scaling_min_freq);
 884 cpufreq_freq_attr_rw(scaling_max_freq);
 885 cpufreq_freq_attr_rw(scaling_governor);
 886 cpufreq_freq_attr_rw(scaling_setspeed);
 887
 888 static struct attribute *default_attrs[] = {
 889         &cpuinfo_min_freq.attr,
 890         &cpuinfo_max_freq.attr,
 891         &cpuinfo_transition_latency.attr,
 892         &scaling_min_freq.attr,
 893         &scaling_max_freq.attr,
 894         &affected_cpus.attr,
 895         &related_cpus.attr,
 896         &scaling_governor.attr,
 897         &scaling_driver.attr,
 898         &scaling_available_governors.attr,
 899         &scaling_setspeed.attr,
 900         NULL
 901 };
 902
 903 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 904 #define to_attr(a) container_of(a, struct freq_attr, attr)
 905
 906 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 907 {
 908         struct cpufreq_policy *policy = to_policy(kobj);
 909         struct freq_attr *fattr = to_attr(attr);
 910         ssize_t ret;
 911
 912         down_read(&policy->rwsem);
 913         ret = fattr->show(policy, buf);
 914         up_read(&policy->rwsem);
 915
 916         return ret;
 917 }
 918
 919 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 920                      const char *buf, size_t count)
 921 {
 922         struct cpufreq_policy *policy = to_policy(kobj);
 923         struct freq_attr *fattr = to_attr(attr);
 924         ssize_t ret = -EINVAL;
 925
 926         /*
 927          * cpus_read_trylock() is used here to work around a circular lock
 928          * dependency problem with respect to the cpufreq_register_driver().
 929          */
 930         if (!cpus_read_trylock())
 931                 return -EBUSY;
 932
 933         if (cpu_online(policy->cpu)) {
 934                 down_write(&policy->rwsem);
 935                 ret = fattr->store(policy, buf, count);
 936                 up_write(&policy->rwsem);
 937         }
 938
 939         cpus_read_unlock();
 940
 941         return ret;
 942 }
 943
 944 static void cpufreq_sysfs_release(struct kobject *kobj)
 945 {
 946         struct cpufreq_policy *policy = to_policy(kobj);
 947         pr_debug("last reference is dropped\n");
 948         complete(&policy->kobj_unregister);
 949 }
 950
 951 static const struct sysfs_ops sysfs_ops = {
 952         .show   = show,
 953         .store  = store,
 954 };
 955
 956 static struct kobj_type ktype_cpufreq = {
 957         .sysfs_ops      = &sysfs_ops,
 958         .default_attrs  = default_attrs,
 959         .release        = cpufreq_sysfs_release,
 960 };
 961
 962 static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
 963 {
 964         struct device *dev = get_cpu_device(cpu);
 965
 966         if (!dev)
 967                 return;
 968
 969         if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
 970                 return;
 971
 972         dev_dbg(dev, "%s: Adding symlink\n", __func__);
 973         if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
 974                 dev_err(dev, "cpufreq symlink creation failed\n");
 975 }
 976
 977 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
 978                                    struct device *dev)
 979 {
 980         dev_dbg(dev, "%s: Removing symlink\n", __func__);
 981         sysfs_remove_link(&dev->kobj, "cpufreq");
 982 }
 983
 984 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
 985 {
 986         struct freq_attr **drv_attr;
 987         int ret = 0;
 988
 989         /* set up files for this cpu device */
 990         drv_attr = cpufreq_driver->attr;
 991         while (drv_attr && *drv_attr) {
 992                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 993                 if (ret)
 994                         return ret;
 995                 drv_attr++;
 996         }
 997         if (cpufreq_driver->get) {
 998                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 999                 if (ret)
1000                         return ret;
1001         }
1002
1003         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1004         if (ret)
1005                 return ret;
1006
1007         if (cpufreq_driver->bios_limit) {
1008                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1009                 if (ret)
1010                         return ret;
1011         }
1012
1013         return 0;
1014 }
1015
1016 __weak struct cpufreq_governor *cpufreq_default_governor(void)
1017 {
1018         return NULL;
1019 }
1020
1021 static int cpufreq_init_policy(struct cpufreq_policy *policy)
1022 {
1023         struct cpufreq_governor *gov = NULL;
1024         struct cpufreq_policy new_policy;
1025
1026         memcpy(&new_policy, policy, sizeof(*policy));
1027
1028         /* Update governor of new_policy to the governor used before hotplug */
1029         gov = find_governor(policy->last_governor);
1030         if (gov) {
1031                 pr_debug("Restoring governor %s for cpu %d\n",
1032                                 policy->governor->name, policy->cpu);
1033         } else {
1034                 gov = cpufreq_default_governor();
1035                 if (!gov)
1036                         return -ENODATA;
1037         }
1038
1039         new_policy.governor = gov;
1040
1041         /* Use the default policy if there is no last_policy. */
1042         if (cpufreq_driver->setpolicy) {
1043                 if (policy->last_policy)
1044                         new_policy.policy = policy->last_policy;
1045                 else
1046                         cpufreq_parse_governor(gov->name, &new_policy);
1047         }
1048         /* set default policy */
1049         return cpufreq_set_policy(policy, &new_policy);
1050 }
1051
1052 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1053 {
1054         int ret = 0;
1055
1056         /* Has this CPU been taken care of already? */
1057         if (cpumask_test_cpu(cpu, policy->cpus))
1058                 return 0;
1059
1060         down_write(&policy->rwsem);
1061         if (has_target())
1062                 cpufreq_stop_governor(policy);
1063
1064         cpumask_set_cpu(cpu, policy->cpus);
1065
1066         if (has_target()) {
1067                 ret = cpufreq_start_governor(policy);
1068                 if (ret)
1069                         pr_err("%s: Failed to start governor\n", __func__);
1070         }
1071         up_write(&policy->rwsem);
1072         return ret;
1073 }
1074
1075 static void handle_update(struct work_struct *work)
1076 {
1077         struct cpufreq_policy *policy =
1078                 container_of(work, struct cpufreq_policy, update);
1079         unsigned int cpu = policy->cpu;
1080         pr_debug("handle_update for cpu %u called\n", cpu);
1081         cpufreq_update_policy(cpu);
1082 }
1083
1084 static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1085 {
1086         struct cpufreq_policy *policy;
1087         int ret;
1088
1089         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1090         if (!policy)
1091                 return NULL;
1092
1093         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1094                 goto err_free_policy;
1095
1096         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1097                 goto err_free_cpumask;
1098
1099         if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1100                 goto err_free_rcpumask;
1101
1102         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1103                                    cpufreq_global_kobject, "policy%u", cpu);
1104         if (ret) {
1105                 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret);
1106                 goto err_free_real_cpus;
1107         }
1108
1109         INIT_LIST_HEAD(&policy->policy_list);
1110         init_rwsem(&policy->rwsem);
1111         spin_lock_init(&policy->transition_lock);
1112         init_waitqueue_head(&policy->transition_wait);
1113         init_completion(&policy->kobj_unregister);
1114         INIT_WORK(&policy->update, handle_update);
1115
1116         policy->cpu = cpu;
1117         return policy;
1118
1119 err_free_real_cpus:
1120         free_cpumask_var(policy->real_cpus);
1121 err_free_rcpumask:
1122         free_cpumask_var(policy->related_cpus);
1123 err_free_cpumask:
1124         free_cpumask_var(policy->cpus);
1125 err_free_policy:
1126         kfree(policy);
1127
1128         return NULL;
1129 }
1130
1131 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1132 {
1133         struct kobject *kobj;
1134         struct completion *cmp;
1135
1136         down_write(&policy->rwsem);
1137         cpufreq_stats_free_table(policy);
1138         kobj = &policy->kobj;
1139         cmp = &policy->kobj_unregister;
1140         up_write(&policy->rwsem);
1141         kobject_put(kobj);
1142
1143         /*
1144          * We need to make sure that the underlying kobj is
1145          * actually not referenced anymore by anybody before we
1146          * proceed with unloading.
1147          */
1148         pr_debug("waiting for dropping of refcount\n");
1149         wait_for_completion(cmp);
1150         pr_debug("wait complete\n");
1151 }
1152
1153 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1154 {
1155         unsigned long flags;
1156         int cpu;
1157
1158         /* Remove policy from list */
1159         write_lock_irqsave(&cpufreq_driver_lock, flags);
1160         list_del(&policy->policy_list);
1161
1162         for_each_cpu(cpu, policy->related_cpus)
1163                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1164         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1165
1166         cpufreq_policy_put_kobj(policy);
1167         free_cpumask_var(policy->real_cpus);
1168         free_cpumask_var(policy->related_cpus);
1169         free_cpumask_var(policy->cpus);
1170         kfree(policy);
1171 }
1172
1173 static int cpufreq_online(unsigned int cpu)
1174 {
1175         struct cpufreq_policy *policy;
1176         bool new_policy;
1177         unsigned long flags;
1178         unsigned int j;
1179         int ret;
1180
1181         pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
1182
1183         /* Check if this CPU already has a policy to manage it */
1184         policy = per_cpu(cpufreq_cpu_data, cpu);
1185         if (policy) {
1186                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1187                 if (!policy_is_inactive(policy))
1188                         return cpufreq_add_policy_cpu(policy, cpu);
1189
1190                 /* This is the only online CPU for the policy.  Start over. */
1191                 new_policy = false;
1192                 down_write(&policy->rwsem);
1193                 policy->cpu = cpu;
1194                 policy->governor = NULL;
1195                 up_write(&policy->rwsem);
1196         } else {
1197                 new_policy = true;
1198                 policy = cpufreq_policy_alloc(cpu);
1199                 if (!policy)
1200                         return -ENOMEM;
1201         }
1202
1203         cpumask_copy(policy->cpus, cpumask_of(cpu));
1204
1205         /* call driver. From then on the cpufreq must be able
1206          * to accept all calls to ->verify and ->setpolicy for this CPU
1207          */
1208         ret = cpufreq_driver->init(policy);
1209         if (ret) {
1210                 pr_debug("initialization failed\n");
1211                 goto out_free_policy;
1212         }
1213
1214         ret = cpufreq_table_validate_and_sort(policy);
1215         if (ret)
1216                 goto out_exit_policy;
1217
1218         down_write(&policy->rwsem);
1219
1220         if (new_policy) {
1221                 /* related_cpus should at least include policy->cpus. */
1222                 cpumask_copy(policy->related_cpus, policy->cpus);
1223         }
1224
1225         /*
1226          * affected cpus must always be the one, which are online. We aren't
1227          * managing offline cpus here.
1228          */
1229         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1230
1231         if (new_policy) {
1232                 policy->user_policy.min = policy->min;
1233                 policy->user_policy.max = policy->max;
1234
1235                 for_each_cpu(j, policy->related_cpus) {
1236                         per_cpu(cpufreq_cpu_data, j) = policy;
1237                         add_cpu_dev_symlink(policy, j);
1238                 }
1239         } else {
1240                 policy->min = policy->user_policy.min;
1241                 policy->max = policy->user_policy.max;
1242         }
1243
1244         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1245                 policy->cur = cpufreq_driver->get(policy->cpu);
1246                 if (!policy->cur) {
1247                         pr_err("%s: ->get() failed\n", __func__);
1248                         goto out_destroy_policy;
1249                 }
1250         }
1251
1252         /*
1253          * Sometimes boot loaders set CPU frequency to a value outside of
1254          * frequency table present with cpufreq core. In such cases CPU might be
1255          * unstable if it has to run on that frequency for long duration of time
1256          * and so its better to set it to a frequency which is specified in
1257          * freq-table. This also makes cpufreq stats inconsistent as
1258          * cpufreq-stats would fail to register because current frequency of CPU
1259          * isn't found in freq-table.
1260          *
1261          * Because we don't want this change to effect boot process badly, we go
1262          * for the next freq which is >= policy->cur ('cur' must be set by now,
1263          * otherwise we will end up setting freq to lowest of the table as 'cur'
1264          * is initialized to zero).
1265          *
1266          * We are passing target-freq as "policy->cur - 1" otherwise
1267          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1268          * equal to target-freq.
1269          */
1270         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1271             && has_target()) {
1272                 /* Are we running at unknown frequency ? */
1273                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1274                 if (ret == -EINVAL) {
1275                         /* Warn user and fix it */
1276                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1277                                 __func__, policy->cpu, policy->cur);
1278                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1279                                 CPUFREQ_RELATION_L);
1280
1281                         /*
1282                          * Reaching here after boot in a few seconds may not
1283                          * mean that system will remain stable at "unknown"
1284                          * frequency for longer duration. Hence, a BUG_ON().
1285                          */
1286                         BUG_ON(ret);
1287                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1288                                 __func__, policy->cpu, policy->cur);
1289                 }
1290         }
1291
1292         if (new_policy) {
1293                 ret = cpufreq_add_dev_interface(policy);
1294                 if (ret)
1295                         goto out_destroy_policy;
1296
1297                 cpufreq_stats_create_table(policy);
1298
1299                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1300                 list_add(&policy->policy_list, &cpufreq_policy_list);
1301                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1302         }
1303
1304         ret = cpufreq_init_policy(policy);
1305         if (ret) {
1306                 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1307                        __func__, cpu, ret);
1308                 goto out_destroy_policy;
1309         }
1310
1311         up_write(&policy->rwsem);
1312
1313         kobject_uevent(&policy->kobj, KOBJ_ADD);
1314
1315         /* Callback for handling stuff after policy is ready */
1316         if (cpufreq_driver->ready)
1317                 cpufreq_driver->ready(policy);
1318
1319         pr_debug("initialization complete\n");
1320
1321         return 0;
1322
1323 out_destroy_policy:
1324         for_each_cpu(j, policy->real_cpus)
1325                 remove_cpu_dev_symlink(policy, get_cpu_device(j));
1326
1327         up_write(&policy->rwsem);
1328
1329 out_exit_policy:
1330         if (cpufreq_driver->exit)
1331                 cpufreq_driver->exit(policy);
1332
1333 out_free_policy:
1334         cpufreq_policy_free(policy);
1335         return ret;
1336 }
1337
1338 /**
1339  * cpufreq_add_dev - the cpufreq interface for a CPU device.
1340  * @dev: CPU device.
1341  * @sif: Subsystem interface structure pointer (not used)
1342  */
1343 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1344 {
1345         struct cpufreq_policy *policy;
1346         unsigned cpu = dev->id;
1347         int ret;
1348
1349         dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1350
1351         if (cpu_online(cpu)) {
1352                 ret = cpufreq_online(cpu);
1353                 if (ret)
1354                         return ret;
1355         }
1356
1357         /* Create sysfs link on CPU registration */
1358         policy = per_cpu(cpufreq_cpu_data, cpu);
1359         if (policy)
1360                 add_cpu_dev_symlink(policy, cpu);
1361
1362         return 0;
1363 }
1364
1365 static int cpufreq_offline(unsigned int cpu)
1366 {
1367         struct cpufreq_policy *policy;
1368         int ret;
1369
1370         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1371
1372         policy = cpufreq_cpu_get_raw(cpu);
1373         if (!policy) {
1374                 pr_debug("%s: No cpu_data found\n", __func__);
1375                 return 0;
1376         }
1377
1378         down_write(&policy->rwsem);
1379         if (has_target())
1380                 cpufreq_stop_governor(policy);
1381
1382         cpumask_clear_cpu(cpu, policy->cpus);
1383
1384         if (policy_is_inactive(policy)) {
1385                 if (has_target())
1386                         strncpy(policy->last_governor, policy->governor->name,
1387                                 CPUFREQ_NAME_LEN);
1388                 else
1389                         policy->last_policy = policy->policy;
1390         } else if (cpu == policy->cpu) {
1391                 /* Nominate new CPU */
1392                 policy->cpu = cpumask_any(policy->cpus);
1393         }
1394
1395         /* Start governor again for active policy */
1396         if (!policy_is_inactive(policy)) {
1397                 if (has_target()) {
1398                         ret = cpufreq_start_governor(policy);
1399                         if (ret)
1400                                 pr_err("%s: Failed to start governor\n", __func__);
1401                 }
1402
1403                 goto unlock;
1404         }
1405
1406         if (cpufreq_driver->stop_cpu)
1407                 cpufreq_driver->stop_cpu(policy);
1408
1409         if (has_target())
1410                 cpufreq_exit_governor(policy);
1411
1412         /*
1413          * Perform the ->exit() even during light-weight tear-down,
1414          * since this is a core component, and is essential for the
1415          * subsequent light-weight ->init() to succeed.
1416          */
1417         if (cpufreq_driver->exit) {
1418                 cpufreq_driver->exit(policy);
1419                 policy->freq_table = NULL;
1420         }
1421
1422 unlock:
1423         up_write(&policy->rwsem);
1424         return 0;
1425 }
1426
1427 /**
1428  * cpufreq_remove_dev - remove a CPU device
1429  *
1430  * Removes the cpufreq interface for a CPU device.
1431  */
1432 static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1433 {
1434         unsigned int cpu = dev->id;
1435         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1436
1437         if (!policy)
1438                 return;
1439
1440         if (cpu_online(cpu))
1441                 cpufreq_offline(cpu);
1442
1443         cpumask_clear_cpu(cpu, policy->real_cpus);
1444         remove_cpu_dev_symlink(policy, dev);
1445
1446         if (cpumask_empty(policy->real_cpus))
1447                 cpufreq_policy_free(policy);
1448 }
1449
1450 /**
1451  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1452  *      in deep trouble.
1453  *      @policy: policy managing CPUs
1454  *      @new_freq: CPU frequency the CPU actually runs at
1455  *
1456  *      We adjust to current frequency first, and need to clean up later.
1457  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1458  */
1459 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1460                                 unsigned int new_freq)
1461 {
1462         struct cpufreq_freqs freqs;
1463
1464         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1465                  policy->cur, new_freq);
1466
1467         freqs.old = policy->cur;
1468         freqs.new = new_freq;
1469
1470         cpufreq_freq_transition_begin(policy, &freqs);
1471         cpufreq_freq_transition_end(policy, &freqs, 0);
1472 }
1473
1474 /**
1475  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1476  * @cpu: CPU number
1477  *
1478  * This is the last known freq, without actually getting it from the driver.
1479  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1480  */
1481 unsigned int cpufreq_quick_get(unsigned int cpu)
1482 {
1483         struct cpufreq_policy *policy;
1484         unsigned int ret_freq = 0;
1485         unsigned long flags;
1486
1487         read_lock_irqsave(&cpufreq_driver_lock, flags);
1488
1489         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1490                 ret_freq = cpufreq_driver->get(cpu);
1491                 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1492                 return ret_freq;
1493         }
1494
1495         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1496
1497         policy = cpufreq_cpu_get(cpu);
1498         if (policy) {
1499                 ret_freq = policy->cur;
1500                 cpufreq_cpu_put(policy);
1501         }
1502
1503         return ret_freq;
1504 }
1505 EXPORT_SYMBOL(cpufreq_quick_get);
1506
1507 /**
1508  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1509  * @cpu: CPU number
1510  *
1511  * Just return the max possible frequency for a given CPU.
1512  */
1513 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1514 {
1515         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1516         unsigned int ret_freq = 0;
1517
1518         if (policy) {
1519                 ret_freq = policy->max;
1520                 cpufreq_cpu_put(policy);
1521         }
1522
1523         return ret_freq;
1524 }
1525 EXPORT_SYMBOL(cpufreq_quick_get_max);
1526
1527 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1528 {
1529         unsigned int ret_freq = 0;
1530
1531         if (unlikely(policy_is_inactive(policy)) || !cpufreq_driver->get)
1532                 return ret_freq;
1533
1534         ret_freq = cpufreq_driver->get(policy->cpu);
1535
1536         /*
1537          * If fast frequency switching is used with the given policy, the check
1538          * against policy->cur is pointless, so skip it in that case too.
1539          */
1540         if (policy->fast_switch_enabled)
1541                 return ret_freq;
1542
1543         if (ret_freq && policy->cur &&
1544                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1545                 /* verify no discrepancy between actual and
1546                                         saved value exists */
1547                 if (unlikely(ret_freq != policy->cur)) {
1548                         cpufreq_out_of_sync(policy, ret_freq);
1549                         schedule_work(&policy->update);
1550                 }
1551         }
1552
1553         return ret_freq;
1554 }
1555
1556 /**
1557  * cpufreq_get - get the current CPU frequency (in kHz)
1558  * @cpu: CPU number
1559  *
1560  * Get the CPU current (static) CPU frequency
1561  */
1562 unsigned int cpufreq_get(unsigned int cpu)
1563 {
1564         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1565         unsigned int ret_freq = 0;
1566
1567         if (policy) {
1568                 down_read(&policy->rwsem);
1569                 ret_freq = __cpufreq_get(policy);
1570                 up_read(&policy->rwsem);
1571
1572                 cpufreq_cpu_put(policy);
1573         }
1574
1575         return ret_freq;
1576 }
1577 EXPORT_SYMBOL(cpufreq_get);
1578
1579 static unsigned int cpufreq_update_current_freq(struct cpufreq_policy *policy)
1580 {
1581         unsigned int new_freq;
1582
1583         new_freq = cpufreq_driver->get(policy->cpu);
1584         if (!new_freq)
1585                 return 0;
1586
1587         if (!policy->cur) {
1588                 pr_debug("cpufreq: Driver did not initialize current freq\n");
1589                 policy->cur = new_freq;
1590         } else if (policy->cur != new_freq && has_target()) {
1591                 cpufreq_out_of_sync(policy, new_freq);
1592         }
1593
1594         return new_freq;
1595 }
1596
1597 static struct subsys_interface cpufreq_interface = {
1598         .name           = "cpufreq",
1599         .subsys         = &cpu_subsys,
1600         .add_dev        = cpufreq_add_dev,
1601         .remove_dev     = cpufreq_remove_dev,
1602 };
1603
1604 /*
1605  * In case platform wants some specific frequency to be configured
1606  * during suspend..
1607  */
1608 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1609 {
1610         int ret;
1611
1612         if (!policy->suspend_freq) {
1613                 pr_debug("%s: suspend_freq not defined\n", __func__);
1614                 return 0;
1615         }
1616
1617         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1618                         policy->suspend_freq);
1619
1620         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1621                         CPUFREQ_RELATION_H);
1622         if (ret)
1623                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1624                                 __func__, policy->suspend_freq, ret);
1625
1626         return ret;
1627 }
1628 EXPORT_SYMBOL(cpufreq_generic_suspend);
1629
1630 /**
1631  * cpufreq_suspend() - Suspend CPUFreq governors
1632  *
1633  * Called during system wide Suspend/Hibernate cycles for suspending governors
1634  * as some platforms can't change frequency after this point in suspend cycle.
1635  * Because some of the devices (like: i2c, regulators, etc) they use for
1636  * changing frequency are suspended quickly after this point.
1637  */
1638 void cpufreq_suspend(void)
1639 {
1640         struct cpufreq_policy *policy;
1641
1642         if (!cpufreq_driver)
1643                 return;
1644
1645         if (!has_target() && !cpufreq_driver->suspend)
1646                 goto suspend;
1647
1648         pr_debug("%s: Suspending Governors\n", __func__);
1649
1650         for_each_active_policy(policy) {
1651                 if (has_target()) {
1652                         down_write(&policy->rwsem);
1653                         cpufreq_stop_governor(policy);
1654                         up_write(&policy->rwsem);
1655                 }
1656
1657                 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
1658                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1659                                 policy);
1660         }
1661
1662 suspend:
1663         cpufreq_suspended = true;
1664 }
1665
1666 /**
1667  * cpufreq_resume() - Resume CPUFreq governors
1668  *
1669  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1670  * are suspended with cpufreq_suspend().
1671  */
1672 void cpufreq_resume(void)
1673 {
1674         struct cpufreq_policy *policy;
1675         int ret;
1676
1677         if (!cpufreq_driver)
1678                 return;
1679
1680         if (unlikely(!cpufreq_suspended))
1681                 return;
1682
1683         cpufreq_suspended = false;
1684
1685         if (!has_target() && !cpufreq_driver->resume)
1686                 return;
1687
1688         pr_debug("%s: Resuming Governors\n", __func__);
1689
1690         for_each_active_policy(policy) {
1691                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
1692                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1693                                 policy);
1694                 } else if (has_target()) {
1695                         down_write(&policy->rwsem);
1696                         ret = cpufreq_start_governor(policy);
1697                         up_write(&policy->rwsem);
1698
1699                         if (ret)
1700                                 pr_err("%s: Failed to start governor for policy: %p\n",
1701                                        __func__, policy);
1702                 }
1703         }
1704 }
1705
1706 /**
1707  *      cpufreq_get_current_driver - return current driver's name
1708  *
1709  *      Return the name string of the currently loaded cpufreq driver
1710  *      or NULL, if none.
1711  */
1712 const char *cpufreq_get_current_driver(void)
1713 {
1714         if (cpufreq_driver)
1715                 return cpufreq_driver->name;
1716
1717         return NULL;
1718 }
1719 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1720
1721 /**
1722  *      cpufreq_get_driver_data - return current driver data
1723  *
1724  *      Return the private data of the currently loaded cpufreq
1725  *      driver, or NULL if no cpufreq driver is loaded.
1726  */
1727 void *cpufreq_get_driver_data(void)
1728 {
1729         if (cpufreq_driver)
1730                 return cpufreq_driver->driver_data;
1731
1732         return NULL;
1733 }
1734 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1735
1736 /*********************************************************************
1737  *                     NOTIFIER LISTS INTERFACE                      *
1738  *********************************************************************/
1739
1740 /**
1741  *      cpufreq_register_notifier - register a driver with cpufreq
1742  *      @nb: notifier function to register
1743  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1744  *
1745  *      Add a driver to one of two lists: either a list of drivers that
1746  *      are notified about clock rate changes (once before and once after
1747  *      the transition), or a list of drivers that are notified about
1748  *      changes in cpufreq policy.
1749  *
1750  *      This function may sleep, and has the same return conditions as
1751  *      blocking_notifier_chain_register.
1752  */
1753 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1754 {
1755         int ret;
1756
1757         if (cpufreq_disabled())
1758                 return -EINVAL;
1759
1760         switch (list) {
1761         case CPUFREQ_TRANSITION_NOTIFIER:
1762                 mutex_lock(&cpufreq_fast_switch_lock);
1763
1764                 if (cpufreq_fast_switch_count > 0) {
1765                         mutex_unlock(&cpufreq_fast_switch_lock);
1766                         return -EBUSY;
1767                 }
1768                 ret = srcu_notifier_chain_register(
1769                                 &cpufreq_transition_notifier_list, nb);
1770                 if (!ret)
1771                         cpufreq_fast_switch_count--;
1772
1773                 mutex_unlock(&cpufreq_fast_switch_lock);
1774                 break;
1775         case CPUFREQ_POLICY_NOTIFIER:
1776                 ret = blocking_notifier_chain_register(
1777                                 &cpufreq_policy_notifier_list, nb);
1778                 break;
1779         default:
1780                 ret = -EINVAL;
1781         }
1782
1783         return ret;
1784 }
1785 EXPORT_SYMBOL(cpufreq_register_notifier);
1786
1787 /**
1788  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1789  *      @nb: notifier block to be unregistered
1790  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1791  *
1792  *      Remove a driver from the CPU frequency notifier list.
1793  *
1794  *      This function may sleep, and has the same return conditions as
1795  *      blocking_notifier_chain_unregister.
1796  */
1797 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1798 {
1799         int ret;
1800
1801         if (cpufreq_disabled())
1802                 return -EINVAL;
1803
1804         switch (list) {
1805         case CPUFREQ_TRANSITION_NOTIFIER:
1806                 mutex_lock(&cpufreq_fast_switch_lock);
1807
1808                 ret = srcu_notifier_chain_unregister(
1809                                 &cpufreq_transition_notifier_list, nb);
1810                 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
1811                         cpufreq_fast_switch_count++;
1812
1813                 mutex_unlock(&cpufreq_fast_switch_lock);
1814                 break;
1815         case CPUFREQ_POLICY_NOTIFIER:
1816                 ret = blocking_notifier_chain_unregister(
1817                                 &cpufreq_policy_notifier_list, nb);
1818                 break;
1819         default:
1820                 ret = -EINVAL;
1821         }
1822
1823         return ret;
1824 }
1825 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1826
1827
1828 /*********************************************************************
1829  *                              GOVERNORS                            *
1830  *********************************************************************/
1831
1832 /**
1833  * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
1834  * @policy: cpufreq policy to switch the frequency for.
1835  * @target_freq: New frequency to set (may be approximate).
1836  *
1837  * Carry out a fast frequency switch without sleeping.
1838  *
1839  * The driver's ->fast_switch() callback invoked by this function must be
1840  * suitable for being called from within RCU-sched read-side critical sections
1841  * and it is expected to select the minimum available frequency greater than or
1842  * equal to @target_freq (CPUFREQ_RELATION_L).
1843  *
1844  * This function must not be called if policy->fast_switch_enabled is unset.
1845  *
1846  * Governors calling this function must guarantee that it will never be invoked
1847  * twice in parallel for the same policy and that it will never be called in
1848  * parallel with either ->target() or ->target_index() for the same policy.
1849  *
1850  * Returns the actual frequency set for the CPU.
1851  *
1852  * If 0 is returned by the driver's ->fast_switch() callback to indicate an
1853  * error condition, the hardware configuration must be preserved.
1854  */
1855 unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
1856                                         unsigned int target_freq)
1857 {
1858         target_freq = clamp_val(target_freq, policy->min, policy->max);
1859
1860         return cpufreq_driver->fast_switch(policy, target_freq);
1861 }
1862 EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
1863
1864 /* Must set freqs->new to intermediate frequency */
1865 static int __target_intermediate(struct cpufreq_policy *policy,
1866                                  struct cpufreq_freqs *freqs, int index)
1867 {
1868         int ret;
1869
1870         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1871
1872         /* We don't need to switch to intermediate freq */
1873         if (!freqs->new)
1874                 return 0;
1875
1876         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1877                  __func__, policy->cpu, freqs->old, freqs->new);
1878
1879         cpufreq_freq_transition_begin(policy, freqs);
1880         ret = cpufreq_driver->target_intermediate(policy, index);
1881         cpufreq_freq_transition_end(policy, freqs, ret);
1882
1883         if (ret)
1884                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1885                        __func__, ret);
1886
1887         return ret;
1888 }
1889
1890 static int __target_index(struct cpufreq_policy *policy, int index)
1891 {
1892         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1893         unsigned int intermediate_freq = 0;
1894         unsigned int newfreq = policy->freq_table[index].frequency;
1895         int retval = -EINVAL;
1896         bool notify;
1897
1898         if (newfreq == policy->cur)
1899                 return 0;
1900
1901         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1902         if (notify) {
1903                 /* Handle switching to intermediate frequency */
1904                 if (cpufreq_driver->get_intermediate) {
1905                         retval = __target_intermediate(policy, &freqs, index);
1906                         if (retval)
1907                                 return retval;
1908
1909                         intermediate_freq = freqs.new;
1910                         /* Set old freq to intermediate */
1911                         if (intermediate_freq)
1912                                 freqs.old = freqs.new;
1913                 }
1914
1915                 freqs.new = newfreq;
1916                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1917                          __func__, policy->cpu, freqs.old, freqs.new);
1918
1919                 cpufreq_freq_transition_begin(policy, &freqs);
1920         }
1921
1922         retval = cpufreq_driver->target_index(policy, index);
1923         if (retval)
1924                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1925                        retval);
1926
1927         if (notify) {
1928                 cpufreq_freq_transition_end(policy, &freqs, retval);
1929
1930                 /*
1931                  * Failed after setting to intermediate freq? Driver should have
1932                  * reverted back to initial frequency and so should we. Check
1933                  * here for intermediate_freq instead of get_intermediate, in
1934                  * case we haven't switched to intermediate freq at all.
1935                  */
1936                 if (unlikely(retval && intermediate_freq)) {
1937                         freqs.old = intermediate_freq;
1938                         freqs.new = policy->restore_freq;
1939                         cpufreq_freq_transition_begin(policy, &freqs);
1940                         cpufreq_freq_transition_end(policy, &freqs, 0);
1941                 }
1942         }
1943
1944         return retval;
1945 }
1946
1947 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1948                             unsigned int target_freq,
1949                             unsigned int relation)
1950 {
1951         unsigned int old_target_freq = target_freq;
1952         int index;
1953
1954         if (cpufreq_disabled())
1955                 return -ENODEV;
1956
1957         /* Make sure that target_freq is within supported range */
1958         target_freq = clamp_val(target_freq, policy->min, policy->max);
1959
1960         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1961                  policy->cpu, target_freq, relation, old_target_freq);
1962
1963         /*
1964          * This might look like a redundant call as we are checking it again
1965          * after finding index. But it is left intentionally for cases where
1966          * exactly same freq is called again and so we can save on few function
1967          * calls.
1968          */
1969         if (target_freq == policy->cur)
1970                 return 0;
1971
1972         /* Save last value to restore later on errors */
1973         policy->restore_freq = policy->cur;
1974
1975         if (cpufreq_driver->target)
1976                 return cpufreq_driver->target(policy, target_freq, relation);
1977
1978         if (!cpufreq_driver->target_index)
1979                 return -EINVAL;
1980
1981         index = cpufreq_frequency_table_target(policy, target_freq, relation);
1982
1983         return __target_index(policy, index);
1984 }
1985 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1986
1987 int cpufreq_driver_target(struct cpufreq_policy *policy,
1988                           unsigned int target_freq,
1989                           unsigned int relation)
1990 {
1991         int ret = -EINVAL;
1992
1993         down_write(&policy->rwsem);
1994
1995         ret = __cpufreq_driver_target(policy, target_freq, relation);
1996
1997         up_write(&policy->rwsem);
1998
1999         return ret;
2000 }
2001 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2002
2003 __weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2004 {
2005         return NULL;
2006 }
2007
2008 static int cpufreq_init_governor(struct cpufreq_policy *policy)
2009 {
2010         int ret;
2011
2012         /* Don't start any governor operations if we are entering suspend */
2013         if (cpufreq_suspended)
2014                 return 0;
2015         /*
2016          * Governor might not be initiated here if ACPI _PPC changed
2017          * notification happened, so check it.
2018          */
2019         if (!policy->governor)
2020                 return -EINVAL;
2021
2022         /* Platform doesn't want dynamic frequency switching ? */
2023         if (policy->governor->dynamic_switching &&
2024             cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
2025                 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2026
2027                 if (gov) {
2028                         pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
2029                                 policy->governor->name, gov->name);
2030                         policy->governor = gov;
2031                 } else {
2032                         return -EINVAL;
2033                 }
2034         }
2035
2036         if (!try_module_get(policy->governor->owner))
2037                 return -EINVAL;
2038
2039         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2040
2041         if (policy->governor->init) {
2042                 ret = policy->governor->init(policy);
2043                 if (ret) {
2044                         module_put(policy->governor->owner);
2045                         return ret;
2046                 }
2047         }
2048
2049         return 0;
2050 }
2051
2052 static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2053 {
2054         if (cpufreq_suspended || !policy->governor)
2055                 return;
2056
2057         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2058
2059         if (policy->governor->exit)
2060                 policy->governor->exit(policy);
2061
2062         module_put(policy->governor->owner);
2063 }
2064
2065 static int cpufreq_start_governor(struct cpufreq_policy *policy)
2066 {
2067         int ret;
2068
2069         if (cpufreq_suspended)
2070                 return 0;
2071
2072         if (!policy->governor)
2073                 return -EINVAL;
2074
2075         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2076
2077         if (cpufreq_driver->get && !cpufreq_driver->setpolicy)
2078                 cpufreq_update_current_freq(policy);
2079
2080         if (policy->governor->start) {
2081                 ret = policy->governor->start(policy);
2082                 if (ret)
2083                         return ret;
2084         }
2085
2086         if (policy->governor->limits)
2087                 policy->governor->limits(policy);
2088
2089         return 0;
2090 }
2091
2092 static void cpufreq_stop_governor(struct cpufreq_policy *policy)
2093 {
2094         if (cpufreq_suspended || !policy->governor)
2095                 return;
2096
2097         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2098
2099         if (policy->governor->stop)
2100                 policy->governor->stop(policy);
2101 }
2102
2103 static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2104 {
2105         if (cpufreq_suspended || !policy->governor)
2106                 return;
2107
2108         pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2109
2110         if (policy->governor->limits)
2111                 policy->governor->limits(policy);
2112 }
2113
2114 int cpufreq_register_governor(struct cpufreq_governor *governor)
2115 {
2116         int err;
2117
2118         if (!governor)
2119                 return -EINVAL;
2120
2121         if (cpufreq_disabled())
2122                 return -ENODEV;
2123
2124         mutex_lock(&cpufreq_governor_mutex);
2125
2126         err = -EBUSY;
2127         if (!find_governor(governor->name)) {
2128                 err = 0;
2129                 list_add(&governor->governor_list, &cpufreq_governor_list);
2130         }
2131
2132         mutex_unlock(&cpufreq_governor_mutex);
2133         return err;
2134 }
2135 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2136
2137 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2138 {
2139         struct cpufreq_policy *policy;
2140         unsigned long flags;
2141
2142         if (!governor)
2143                 return;
2144
2145         if (cpufreq_disabled())
2146                 return;
2147
2148         /* clear last_governor for all inactive policies */
2149         read_lock_irqsave(&cpufreq_driver_lock, flags);
2150         for_each_inactive_policy(policy) {
2151                 if (!strcmp(policy->last_governor, governor->name)) {
2152                         policy->governor = NULL;
2153                         strcpy(policy->last_governor, "\0");
2154                 }
2155         }
2156         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2157
2158         mutex_lock(&cpufreq_governor_mutex);
2159         list_del(&governor->governor_list);
2160         mutex_unlock(&cpufreq_governor_mutex);
2161 }
2162 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2163
2164
2165 /*********************************************************************
2166  *                          POLICY INTERFACE                         *
2167  *********************************************************************/
2168
2169 /**
2170  * cpufreq_get_policy - get the current cpufreq_policy
2171  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2172  *      is written
2173  *
2174  * Reads the current cpufreq policy.
2175  */
2176 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2177 {
2178         struct cpufreq_policy *cpu_policy;
2179         if (!policy)
2180                 return -EINVAL;
2181
2182         cpu_policy = cpufreq_cpu_get(cpu);
2183         if (!cpu_policy)
2184                 return -EINVAL;
2185
2186         memcpy(policy, cpu_policy, sizeof(*policy));
2187
2188         cpufreq_cpu_put(cpu_policy);
2189         return 0;
2190 }
2191 EXPORT_SYMBOL(cpufreq_get_policy);
2192
2193 /*
2194  * policy : current policy.
2195  * new_policy: policy to be set.
2196  */
2197 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2198                                 struct cpufreq_policy *new_policy)
2199 {
2200         struct cpufreq_governor *old_gov;
2201         int ret;
2202
2203         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2204                  new_policy->cpu, new_policy->min, new_policy->max);
2205
2206         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2207
2208         /*
2209         * This check works well when we store new min/max freq attributes,
2210         * because new_policy is a copy of policy with one field updated.
2211         */
2212         if (new_policy->min > new_policy->max)
2213                 return -EINVAL;
2214
2215         /* verify the cpu speed can be set within this limit */
2216         ret = cpufreq_driver->verify(new_policy);
2217         if (ret)
2218                 return ret;
2219
2220         /* adjust if necessary - all reasons */
2221         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2222                         CPUFREQ_ADJUST, new_policy);
2223
2224         /*
2225          * verify the cpu speed can be set within this limit, which might be
2226          * different to the first one
2227          */
2228         ret = cpufreq_driver->verify(new_policy);
2229         if (ret)
2230                 return ret;
2231
2232         /* notification of the new policy */
2233         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2234                         CPUFREQ_NOTIFY, new_policy);
2235
2236         policy->min = new_policy->min;
2237         policy->max = new_policy->max;
2238         trace_cpu_frequency_limits(policy);
2239
2240         policy->cached_target_freq = UINT_MAX;
2241
2242         pr_debug("new min and max freqs are %u - %u kHz\n",
2243                  policy->min, policy->max);
2244
2245         if (cpufreq_driver->setpolicy) {
2246                 policy->policy = new_policy->policy;
2247                 pr_debug("setting range\n");
2248                 return cpufreq_driver->setpolicy(new_policy);
2249         }
2250
2251         if (new_policy->governor == policy->governor) {
2252                 pr_debug("cpufreq: governor limits update\n");
2253                 cpufreq_governor_limits(policy);
2254                 return 0;
2255         }
2256
2257         pr_debug("governor switch\n");
2258
2259         /* save old, working values */
2260         old_gov = policy->governor;
2261         /* end old governor */
2262         if (old_gov) {
2263                 cpufreq_stop_governor(policy);
2264                 cpufreq_exit_governor(policy);
2265         }
2266
2267         /* start new governor */
2268         policy->governor = new_policy->governor;
2269         ret = cpufreq_init_governor(policy);
2270         if (!ret) {
2271                 ret = cpufreq_start_governor(policy);
2272                 if (!ret) {
2273                         pr_debug("cpufreq: governor change\n");
2274                         sched_cpufreq_governor_change(policy, old_gov);
2275                         return 0;
2276                 }
2277                 cpufreq_exit_governor(policy);
2278         }
2279
2280         /* new governor failed, so re-start old one */
2281         pr_debug("starting governor %s failed\n", policy->governor->name);
2282         if (old_gov) {
2283                 policy->governor = old_gov;
2284                 if (cpufreq_init_governor(policy))
2285                         policy->governor = NULL;
2286                 else
2287                         cpufreq_start_governor(policy);
2288         }
2289
2290         return ret;
2291 }
2292
2293 /**
2294  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2295  *      @cpu: CPU which shall be re-evaluated
2296  *
2297  *      Useful for policy notifiers which have different necessities
2298  *      at different times.
2299  */
2300 void cpufreq_update_policy(unsigned int cpu)
2301 {
2302         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2303         struct cpufreq_policy new_policy;
2304
2305         if (!policy)
2306                 return;
2307
2308         down_write(&policy->rwsem);
2309
2310         if (policy_is_inactive(policy))
2311                 goto unlock;
2312
2313         pr_debug("updating policy for CPU %u\n", cpu);
2314         memcpy(&new_policy, policy, sizeof(*policy));
2315         new_policy.min = policy->user_policy.min;
2316         new_policy.max = policy->user_policy.max;
2317
2318         /*
2319          * BIOS might change freq behind our back
2320          * -> ask driver for current freq and notify governors about a change
2321          */
2322         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2323                 if (cpufreq_suspended)
2324                         goto unlock;
2325
2326                 new_policy.cur = cpufreq_update_current_freq(policy);
2327                 if (WARN_ON(!new_policy.cur))
2328                         goto unlock;
2329         }
2330
2331         cpufreq_set_policy(policy, &new_policy);
2332
2333 unlock:
2334         up_write(&policy->rwsem);
2335
2336         cpufreq_cpu_put(policy);
2337 }
2338 EXPORT_SYMBOL(cpufreq_update_policy);
2339
2340 /*********************************************************************
2341  *               BOOST                                               *
2342  *********************************************************************/
2343 static int cpufreq_boost_set_sw(int state)
2344 {
2345         struct cpufreq_policy *policy;
2346         int ret = -EINVAL;
2347
2348         for_each_active_policy(policy) {
2349                 if (!policy->freq_table)
2350                         continue;
2351
2352                 ret = cpufreq_frequency_table_cpuinfo(policy,
2353                                                       policy->freq_table);
2354                 if (ret) {
2355                         pr_err("%s: Policy frequency update failed\n",
2356                                __func__);
2357                         break;
2358                 }
2359
2360                 down_write(&policy->rwsem);
2361                 policy->user_policy.max = policy->max;
2362                 cpufreq_governor_limits(policy);
2363                 up_write(&policy->rwsem);
2364         }
2365
2366         return ret;
2367 }
2368
2369 int cpufreq_boost_trigger_state(int state)
2370 {
2371         unsigned long flags;
2372         int ret = 0;
2373
2374         if (cpufreq_driver->boost_enabled == state)
2375                 return 0;
2376
2377         write_lock_irqsave(&cpufreq_driver_lock, flags);
2378         cpufreq_driver->boost_enabled = state;
2379         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2380
2381         ret = cpufreq_driver->set_boost(state);
2382         if (ret) {
2383                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2384                 cpufreq_driver->boost_enabled = !state;
2385                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2386
2387                 pr_err("%s: Cannot %s BOOST\n",
2388                        __func__, state ? "enable" : "disable");
2389         }
2390
2391         return ret;
2392 }
2393
2394 static bool cpufreq_boost_supported(void)
2395 {
2396         return likely(cpufreq_driver) && cpufreq_driver->set_boost;
2397 }
2398
2399 static int create_boost_sysfs_file(void)
2400 {
2401         int ret;
2402
2403         ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
2404         if (ret)
2405                 pr_err("%s: cannot register global BOOST sysfs file\n",
2406                        __func__);
2407
2408         return ret;
2409 }
2410
2411 static void remove_boost_sysfs_file(void)
2412 {
2413         if (cpufreq_boost_supported())
2414                 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
2415 }
2416
2417 int cpufreq_enable_boost_support(void)
2418 {
2419         if (!cpufreq_driver)
2420                 return -EINVAL;
2421
2422         if (cpufreq_boost_supported())
2423                 return 0;
2424
2425         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2426
2427         /* This will get removed on driver unregister */
2428         return create_boost_sysfs_file();
2429 }
2430 EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2431
2432 int cpufreq_boost_enabled(void)
2433 {
2434         return cpufreq_driver->boost_enabled;
2435 }
2436 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2437
2438 /*********************************************************************
2439  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2440  *********************************************************************/
2441 static enum cpuhp_state hp_online;
2442
2443 static int cpuhp_cpufreq_online(unsigned int cpu)
2444 {
2445         cpufreq_online(cpu);
2446
2447         return 0;
2448 }
2449
2450 static int cpuhp_cpufreq_offline(unsigned int cpu)
2451 {
2452         cpufreq_offline(cpu);
2453
2454         return 0;
2455 }
2456
2457 /**
2458  * cpufreq_register_driver - register a CPU Frequency driver
2459  * @driver_data: A struct cpufreq_driver containing the values#
2460  * submitted by the CPU Frequency driver.
2461  *
2462  * Registers a CPU Frequency driver to this core code. This code
2463  * returns zero on success, -EEXIST when another driver got here first
2464  * (and isn't unregistered in the meantime).
2465  *
2466  */
2467 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2468 {
2469         unsigned long flags;
2470         int ret;
2471
2472         if (cpufreq_disabled())
2473                 return -ENODEV;
2474
2475         if (!driver_data || !driver_data->verify || !driver_data->init ||
2476             !(driver_data->setpolicy || driver_data->target_index ||
2477                     driver_data->target) ||
2478              (driver_data->setpolicy && (driver_data->target_index ||
2479                     driver_data->target)) ||
2480              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2481                 return -EINVAL;
2482
2483         pr_debug("trying to register driver %s\n", driver_data->name);
2484
2485         /* Protect against concurrent CPU online/offline. */
2486         cpus_read_lock();
2487
2488         write_lock_irqsave(&cpufreq_driver_lock, flags);
2489         if (cpufreq_driver) {
2490                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2491                 ret = -EEXIST;
2492                 goto out;
2493         }
2494         cpufreq_driver = driver_data;
2495         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2496
2497         if (driver_data->setpolicy)
2498                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2499
2500         if (cpufreq_boost_supported()) {
2501                 ret = create_boost_sysfs_file();
2502                 if (ret)
2503                         goto err_null_driver;
2504         }
2505
2506         ret = subsys_interface_register(&cpufreq_interface);
2507         if (ret)
2508                 goto err_boost_unreg;
2509
2510         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2511             list_empty(&cpufreq_policy_list)) {
2512                 /* if all ->init() calls failed, unregister */
2513                 ret = -ENODEV;
2514                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2515                          driver_data->name);
2516                 goto err_if_unreg;
2517         }
2518
2519         ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2520                                                    "cpufreq:online",
2521                                                    cpuhp_cpufreq_online,
2522                                                    cpuhp_cpufreq_offline);
2523         if (ret < 0)
2524                 goto err_if_unreg;
2525         hp_online = ret;
2526         ret = 0;
2527
2528         pr_debug("driver %s up and running\n", driver_data->name);
2529         goto out;
2530
2531 err_if_unreg:
2532         subsys_interface_unregister(&cpufreq_interface);
2533 err_boost_unreg:
2534         remove_boost_sysfs_file();
2535 err_null_driver:
2536         write_lock_irqsave(&cpufreq_driver_lock, flags);
2537         cpufreq_driver = NULL;
2538         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2539 out:
2540         cpus_read_unlock();
2541         return ret;
2542 }
2543 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2544
2545 /**
2546  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2547  *
2548  * Unregister the current CPUFreq driver. Only call this if you have
2549  * the right to do so, i.e. if you have succeeded in initialising before!
2550  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2551  * currently not initialised.
2552  */
2553 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2554 {
2555         unsigned long flags;
2556
2557         if (!cpufreq_driver || (driver != cpufreq_driver))
2558                 return -EINVAL;
2559
2560         pr_debug("unregistering driver %s\n", driver->name);
2561
2562         /* Protect against concurrent cpu hotplug */
2563         cpus_read_lock();
2564         subsys_interface_unregister(&cpufreq_interface);
2565         remove_boost_sysfs_file();
2566         cpuhp_remove_state_nocalls_cpuslocked(hp_online);
2567
2568         write_lock_irqsave(&cpufreq_driver_lock, flags);
2569
2570         cpufreq_driver = NULL;
2571
2572         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2573         cpus_read_unlock();
2574
2575         return 0;
2576 }
2577 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2578
2579 /*
2580  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2581  * or mutexes when secondary CPUs are halted.
2582  */
2583 static struct syscore_ops cpufreq_syscore_ops = {
2584         .shutdown = cpufreq_suspend,
2585 };
2586
2587 struct kobject *cpufreq_global_kobject;
2588 EXPORT_SYMBOL(cpufreq_global_kobject);
2589
2590 static int __init cpufreq_core_init(void)
2591 {
2592         if (cpufreq_disabled())
2593                 return -ENODEV;
2594
2595         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2596         BUG_ON(!cpufreq_global_kobject);
2597
2598         register_syscore_ops(&cpufreq_syscore_ops);
2599
2600         return 0;
2601 }
2602 module_param(off, int, 0444);
2603 core_initcall(cpufreq_core_init);