drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 static LIST_HEAD(cpufreq_policy_list);
  35
  36 static inline bool policy_is_inactive(struct cpufreq_policy *policy)
  37 {
  38         return cpumask_empty(policy->cpus);
  39 }
  40
  41 static bool suitable_policy(struct cpufreq_policy *policy, bool active)
  42 {
  43         return active == !policy_is_inactive(policy);
  44 }
  45
  46 /* Finds Next Acive/Inactive policy */
  47 static struct cpufreq_policy *next_policy(struct cpufreq_policy *policy,
  48                                           bool active)
  49 {
  50         do {
  51                 policy = list_next_entry(policy, policy_list);
  52
  53                 /* No more policies in the list */
  54                 if (&policy->policy_list == &cpufreq_policy_list)
  55                         return NULL;
  56         } while (!suitable_policy(policy, active));
  57
  58         return policy;
  59 }
  60
  61 static struct cpufreq_policy *first_policy(bool active)
  62 {
  63         struct cpufreq_policy *policy;
  64
  65         /* No policies in the list */
  66         if (list_empty(&cpufreq_policy_list))
  67                 return NULL;
  68
  69         policy = list_first_entry(&cpufreq_policy_list, typeof(*policy),
  70                                   policy_list);
  71
  72         if (!suitable_policy(policy, active))
  73                 policy = next_policy(policy, active);
  74
  75         return policy;
  76 }
  77
  78 /* Macros to iterate over CPU policies */
  79 #define for_each_suitable_policy(__policy, __active)    \
  80         for (__policy = first_policy(__active);         \
  81              __policy;                                  \
  82              __policy = next_policy(__policy, __active))
  83
  84 #define for_each_active_policy(__policy)                \
  85         for_each_suitable_policy(__policy, true)
  86 #define for_each_inactive_policy(__policy)              \
  87         for_each_suitable_policy(__policy, false)
  88
  89 #define for_each_policy(__policy)                       \
  90         list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
  91
  92 /* Iterate over governors */
  93 static LIST_HEAD(cpufreq_governor_list);
  94 #define for_each_governor(__governor)                           \
  95         list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
  96
  97 /**
  98  * The "cpufreq driver" - the arch- or hardware-dependent low
  99  * level driver of CPUFreq support, and its spinlock. This lock
 100  * also protects the cpufreq_cpu_data array.
 101  */
 102 static struct cpufreq_driver *cpufreq_driver;
 103 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
 104 static DEFINE_RWLOCK(cpufreq_driver_lock);
 105 DEFINE_MUTEX(cpufreq_governor_lock);
 106
 107 /* Flag to suspend/resume CPUFreq governors */
 108 static bool cpufreq_suspended;
 109
 110 static inline bool has_target(void)
 111 {
 112         return cpufreq_driver->target_index || cpufreq_driver->target;
 113 }
 114
 115 /*
 116  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
 117  * sections
 118  */
 119 static DECLARE_RWSEM(cpufreq_rwsem);
 120
 121 /* internal prototypes */
 122 static int __cpufreq_governor(struct cpufreq_policy *policy,
 123                 unsigned int event);
 124 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
 125 static void handle_update(struct work_struct *work);
 126
 127 /**
 128  * Two notifier lists: the "policy" list is involved in the
 129  * validation process for a new CPU frequency policy; the
 130  * "transition" list for kernel code that needs to handle
 131  * changes to devices when the CPU clock speed changes.
 132  * The mutex locks both lists.
 133  */
 134 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 135 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 136
 137 static bool init_cpufreq_transition_notifier_list_called;
 138 static int __init init_cpufreq_transition_notifier_list(void)
 139 {
 140         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 141         init_cpufreq_transition_notifier_list_called = true;
 142         return 0;
 143 }
 144 pure_initcall(init_cpufreq_transition_notifier_list);
 145
 146 static int off __read_mostly;
 147 static int cpufreq_disabled(void)
 148 {
 149         return off;
 150 }
 151 void disable_cpufreq(void)
 152 {
 153         off = 1;
 154 }
 155 static DEFINE_MUTEX(cpufreq_governor_mutex);
 156
 157 bool have_governor_per_policy(void)
 158 {
 159         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 160 }
 161 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 162
 163 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 164 {
 165         if (have_governor_per_policy())
 166                 return &policy->kobj;
 167         else
 168                 return cpufreq_global_kobject;
 169 }
 170 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 171
 172 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 173 {
 174         u64 idle_time;
 175         u64 cur_wall_time;
 176         u64 busy_time;
 177
 178         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 179
 180         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 181         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 182         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 183         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 184         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 185         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 186
 187         idle_time = cur_wall_time - busy_time;
 188         if (wall)
 189                 *wall = cputime_to_usecs(cur_wall_time);
 190
 191         return cputime_to_usecs(idle_time);
 192 }
 193
 194 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 195 {
 196         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 197
 198         if (idle_time == -1ULL)
 199                 return get_cpu_idle_time_jiffy(cpu, wall);
 200         else if (!io_busy)
 201                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 202
 203         return idle_time;
 204 }
 205 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 206
 207 /*
 208  * This is a generic cpufreq init() routine which can be used by cpufreq
 209  * drivers of SMP systems. It will do following:
 210  * - validate & show freq table passed
 211  * - set policies transition latency
 212  * - policy->cpus with all possible CPUs
 213  */
 214 int cpufreq_generic_init(struct cpufreq_policy *policy,
 215                 struct cpufreq_frequency_table *table,
 216                 unsigned int transition_latency)
 217 {
 218         int ret;
 219
 220         ret = cpufreq_table_validate_and_show(policy, table);
 221         if (ret) {
 222                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 223                 return ret;
 224         }
 225
 226         policy->cpuinfo.transition_latency = transition_latency;
 227
 228         /*
 229          * The driver only supports the SMP configuration where all processors
 230          * share the clock and voltage and clock.
 231          */
 232         cpumask_setall(policy->cpus);
 233
 234         return 0;
 235 }
 236 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 237
 238 /* Only for cpufreq core internal use */
 239 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 240 {
 241         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 242
 243         return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
 244 }
 245
 246 unsigned int cpufreq_generic_get(unsigned int cpu)
 247 {
 248         struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
 249
 250         if (!policy || IS_ERR(policy->clk)) {
 251                 pr_err("%s: No %s associated to cpu: %d\n",
 252                        __func__, policy ? "clk" : "policy", cpu);
 253                 return 0;
 254         }
 255
 256         return clk_get_rate(policy->clk) / 1000;
 257 }
 258 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 259
 260 /**
 261  * cpufreq_cpu_get: returns policy for a cpu and marks it busy.
 262  *
 263  * @cpu: cpu to find policy for.
 264  *
 265  * This returns policy for 'cpu', returns NULL if it doesn't exist.
 266  * It also increments the kobject reference count to mark it busy and so would
 267  * require a corresponding call to cpufreq_cpu_put() to decrement it back.
 268  * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be
 269  * freed as that depends on the kobj count.
 270  *
 271  * It also takes a read-lock of 'cpufreq_rwsem' and doesn't put it back if a
 272  * valid policy is found. This is done to make sure the driver doesn't get
 273  * unregistered while the policy is being used.
 274  *
 275  * Return: A valid policy on success, otherwise NULL on failure.
 276  */
 277 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 278 {
 279         struct cpufreq_policy *policy = NULL;
 280         unsigned long flags;
 281
 282         if (WARN_ON(cpu >= nr_cpu_ids))
 283                 return NULL;
 284
 285         if (!down_read_trylock(&cpufreq_rwsem))
 286                 return NULL;
 287
 288         /* get the cpufreq driver */
 289         read_lock_irqsave(&cpufreq_driver_lock, flags);
 290
 291         if (cpufreq_driver) {
 292                 /* get the CPU */
 293                 policy = cpufreq_cpu_get_raw(cpu);
 294                 if (policy)
 295                         kobject_get(&policy->kobj);
 296         }
 297
 298         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 299
 300         if (!policy)
 301                 up_read(&cpufreq_rwsem);
 302
 303         return policy;
 304 }
 305 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 306
 307 /**
 308  * cpufreq_cpu_put: Decrements the usage count of a policy
 309  *
 310  * @policy: policy earlier returned by cpufreq_cpu_get().
 311  *
 312  * This decrements the kobject reference count incremented earlier by calling
 313  * cpufreq_cpu_get().
 314  *
 315  * It also drops the read-lock of 'cpufreq_rwsem' taken at cpufreq_cpu_get().
 316  */
 317 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 318 {
 319         kobject_put(&policy->kobj);
 320         up_read(&cpufreq_rwsem);
 321 }
 322 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 323
 324 /*********************************************************************
 325  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 326  *********************************************************************/
 327
 328 /**
 329  * adjust_jiffies - adjust the system "loops_per_jiffy"
 330  *
 331  * This function alters the system "loops_per_jiffy" for the clock
 332  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 333  * systems as each CPU might be scaled differently. So, use the arch
 334  * per-CPU loops_per_jiffy value wherever possible.
 335  */
 336 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 337 {
 338 #ifndef CONFIG_SMP
 339         static unsigned long l_p_j_ref;
 340         static unsigned int l_p_j_ref_freq;
 341
 342         if (ci->flags & CPUFREQ_CONST_LOOPS)
 343                 return;
 344
 345         if (!l_p_j_ref_freq) {
 346                 l_p_j_ref = loops_per_jiffy;
 347                 l_p_j_ref_freq = ci->old;
 348                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 349                          l_p_j_ref, l_p_j_ref_freq);
 350         }
 351         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 352                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 353                                                                 ci->new);
 354                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 355                          loops_per_jiffy, ci->new);
 356         }
 357 #endif
 358 }
 359
 360 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 361                 struct cpufreq_freqs *freqs, unsigned int state)
 362 {
 363         BUG_ON(irqs_disabled());
 364
 365         if (cpufreq_disabled())
 366                 return;
 367
 368         freqs->flags = cpufreq_driver->flags;
 369         pr_debug("notification %u of frequency transition to %u kHz\n",
 370                  state, freqs->new);
 371
 372         switch (state) {
 373
 374         case CPUFREQ_PRECHANGE:
 375                 /* detect if the driver reported a value as "old frequency"
 376                  * which is not equal to what the cpufreq core thinks is
 377                  * "old frequency".
 378                  */
 379                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 380                         if ((policy) && (policy->cpu == freqs->cpu) &&
 381                             (policy->cur) && (policy->cur != freqs->old)) {
 382                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 383                                          freqs->old, policy->cur);
 384                                 freqs->old = policy->cur;
 385                         }
 386                 }
 387                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 388                                 CPUFREQ_PRECHANGE, freqs);
 389                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 390                 break;
 391
 392         case CPUFREQ_POSTCHANGE:
 393                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 394                 pr_debug("FREQ: %lu - CPU: %lu\n",
 395                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 396                 trace_cpu_frequency(freqs->new, freqs->cpu);
 397                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 398                                 CPUFREQ_POSTCHANGE, freqs);
 399                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 400                         policy->cur = freqs->new;
 401                 break;
 402         }
 403 }
 404
 405 /**
 406  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 407  * on frequency transition.
 408  *
 409  * This function calls the transition notifiers and the "adjust_jiffies"
 410  * function. It is called twice on all CPU frequency changes that have
 411  * external effects.
 412  */
 413 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 414                 struct cpufreq_freqs *freqs, unsigned int state)
 415 {
 416         for_each_cpu(freqs->cpu, policy->cpus)
 417                 __cpufreq_notify_transition(policy, freqs, state);
 418 }
 419
 420 /* Do post notifications when there are chances that transition has failed */
 421 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 422                 struct cpufreq_freqs *freqs, int transition_failed)
 423 {
 424         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 425         if (!transition_failed)
 426                 return;
 427
 428         swap(freqs->old, freqs->new);
 429         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 430         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 431 }
 432
 433 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 434                 struct cpufreq_freqs *freqs)
 435 {
 436
 437         /*
 438          * Catch double invocations of _begin() which lead to self-deadlock.
 439          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 440          * doesn't invoke _begin() on their behalf, and hence the chances of
 441          * double invocations are very low. Moreover, there are scenarios
 442          * where these checks can emit false-positive warnings in these
 443          * drivers; so we avoid that by skipping them altogether.
 444          */
 445         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 446                                 && current == policy->transition_task);
 447
 448 wait:
 449         wait_event(policy->transition_wait, !policy->transition_ongoing);
 450
 451         spin_lock(&policy->transition_lock);
 452
 453         if (unlikely(policy->transition_ongoing)) {
 454                 spin_unlock(&policy->transition_lock);
 455                 goto wait;
 456         }
 457
 458         policy->transition_ongoing = true;
 459         policy->transition_task = current;
 460
 461         spin_unlock(&policy->transition_lock);
 462
 463         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 464 }
 465 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 466
 467 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 468                 struct cpufreq_freqs *freqs, int transition_failed)
 469 {
 470         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 471                 return;
 472
 473         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 474
 475         policy->transition_ongoing = false;
 476         policy->transition_task = NULL;
 477
 478         wake_up(&policy->transition_wait);
 479 }
 480 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 481
 482
 483 /*********************************************************************
 484  *                          SYSFS INTERFACE                          *
 485  *********************************************************************/
 486 static ssize_t show_boost(struct kobject *kobj,
 487                                  struct attribute *attr, char *buf)
 488 {
 489         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 490 }
 491
 492 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 493                                   const char *buf, size_t count)
 494 {
 495         int ret, enable;
 496
 497         ret = sscanf(buf, "%d", &enable);
 498         if (ret != 1 || enable < 0 || enable > 1)
 499                 return -EINVAL;
 500
 501         if (cpufreq_boost_trigger_state(enable)) {
 502                 pr_err("%s: Cannot %s BOOST!\n",
 503                        __func__, enable ? "enable" : "disable");
 504                 return -EINVAL;
 505         }
 506
 507         pr_debug("%s: cpufreq BOOST %s\n",
 508                  __func__, enable ? "enabled" : "disabled");
 509
 510         return count;
 511 }
 512 define_one_global_rw(boost);
 513
 514 static struct cpufreq_governor *find_governor(const char *str_governor)
 515 {
 516         struct cpufreq_governor *t;
 517
 518         for_each_governor(t)
 519                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 520                         return t;
 521
 522         return NULL;
 523 }
 524
 525 /**
 526  * cpufreq_parse_governor - parse a governor string
 527  */
 528 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 529                                 struct cpufreq_governor **governor)
 530 {
 531         int err = -EINVAL;
 532
 533         if (!cpufreq_driver)
 534                 goto out;
 535
 536         if (cpufreq_driver->setpolicy) {
 537                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 538                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 539                         err = 0;
 540                 } else if (!strncasecmp(str_governor, "powersave",
 541                                                 CPUFREQ_NAME_LEN)) {
 542                         *policy = CPUFREQ_POLICY_POWERSAVE;
 543                         err = 0;
 544                 }
 545         } else {
 546                 struct cpufreq_governor *t;
 547
 548                 mutex_lock(&cpufreq_governor_mutex);
 549
 550                 t = find_governor(str_governor);
 551
 552                 if (t == NULL) {
 553                         int ret;
 554
 555                         mutex_unlock(&cpufreq_governor_mutex);
 556                         ret = request_module("cpufreq_%s", str_governor);
 557                         mutex_lock(&cpufreq_governor_mutex);
 558
 559                         if (ret == 0)
 560                                 t = find_governor(str_governor);
 561                 }
 562
 563                 if (t != NULL) {
 564                         *governor = t;
 565                         err = 0;
 566                 }
 567
 568                 mutex_unlock(&cpufreq_governor_mutex);
 569         }
 570 out:
 571         return err;
 572 }
 573
 574 /**
 575  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 576  * print out cpufreq information
 577  *
 578  * Write out information from cpufreq_driver->policy[cpu]; object must be
 579  * "unsigned int".
 580  */
 581
 582 #define show_one(file_name, object)                     \
 583 static ssize_t show_##file_name                         \
 584 (struct cpufreq_policy *policy, char *buf)              \
 585 {                                                       \
 586         return sprintf(buf, "%u\n", policy->object);    \
 587 }
 588
 589 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 590 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 591 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 592 show_one(scaling_min_freq, min);
 593 show_one(scaling_max_freq, max);
 594
 595 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 596 {
 597         ssize_t ret;
 598
 599         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 600                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 601         else
 602                 ret = sprintf(buf, "%u\n", policy->cur);
 603         return ret;
 604 }
 605
 606 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 607                                 struct cpufreq_policy *new_policy);
 608
 609 /**
 610  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 611  */
 612 #define store_one(file_name, object)                    \
 613 static ssize_t store_##file_name                                        \
 614 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 615 {                                                                       \
 616         int ret, temp;                                                  \
 617         struct cpufreq_policy new_policy;                               \
 618                                                                         \
 619         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 620         if (ret)                                                        \
 621                 return -EINVAL;                                         \
 622                                                                         \
 623         ret = sscanf(buf, "%u", &new_policy.object);                    \
 624         if (ret != 1)                                                   \
 625                 return -EINVAL;                                         \
 626                                                                         \
 627         temp = new_policy.object;                                       \
 628         ret = cpufreq_set_policy(policy, &new_policy);          \
 629         if (!ret)                                                       \
 630                 policy->user_policy.object = temp;                      \
 631                                                                         \
 632         return ret ? ret : count;                                       \
 633 }
 634
 635 store_one(scaling_min_freq, min);
 636 store_one(scaling_max_freq, max);
 637
 638 /**
 639  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 640  */
 641 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 642                                         char *buf)
 643 {
 644         unsigned int cur_freq = __cpufreq_get(policy);
 645         if (!cur_freq)
 646                 return sprintf(buf, "<unknown>");
 647         return sprintf(buf, "%u\n", cur_freq);
 648 }
 649
 650 /**
 651  * show_scaling_governor - show the current policy for the specified CPU
 652  */
 653 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 654 {
 655         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 656                 return sprintf(buf, "powersave\n");
 657         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 658                 return sprintf(buf, "performance\n");
 659         else if (policy->governor)
 660                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 661                                 policy->governor->name);
 662         return -EINVAL;
 663 }
 664
 665 /**
 666  * store_scaling_governor - store policy for the specified CPU
 667  */
 668 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 669                                         const char *buf, size_t count)
 670 {
 671         int ret;
 672         char    str_governor[16];
 673         struct cpufreq_policy new_policy;
 674
 675         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 676         if (ret)
 677                 return ret;
 678
 679         ret = sscanf(buf, "%15s", str_governor);
 680         if (ret != 1)
 681                 return -EINVAL;
 682
 683         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 684                                                 &new_policy.governor))
 685                 return -EINVAL;
 686
 687         ret = cpufreq_set_policy(policy, &new_policy);
 688
 689         policy->user_policy.policy = policy->policy;
 690         policy->user_policy.governor = policy->governor;
 691
 692         if (ret)
 693                 return ret;
 694         else
 695                 return count;
 696 }
 697
 698 /**
 699  * show_scaling_driver - show the cpufreq driver currently loaded
 700  */
 701 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 702 {
 703         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 704 }
 705
 706 /**
 707  * show_scaling_available_governors - show the available CPUfreq governors
 708  */
 709 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 710                                                 char *buf)
 711 {
 712         ssize_t i = 0;
 713         struct cpufreq_governor *t;
 714
 715         if (!has_target()) {
 716                 i += sprintf(buf, "performance powersave");
 717                 goto out;
 718         }
 719
 720         for_each_governor(t) {
 721                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 722                     - (CPUFREQ_NAME_LEN + 2)))
 723                         goto out;
 724                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 725         }
 726 out:
 727         i += sprintf(&buf[i], "\n");
 728         return i;
 729 }
 730
 731 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 732 {
 733         ssize_t i = 0;
 734         unsigned int cpu;
 735
 736         for_each_cpu(cpu, mask) {
 737                 if (i)
 738                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 739                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 740                 if (i >= (PAGE_SIZE - 5))
 741                         break;
 742         }
 743         i += sprintf(&buf[i], "\n");
 744         return i;
 745 }
 746 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 747
 748 /**
 749  * show_related_cpus - show the CPUs affected by each transition even if
 750  * hw coordination is in use
 751  */
 752 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 753 {
 754         return cpufreq_show_cpus(policy->related_cpus, buf);
 755 }
 756
 757 /**
 758  * show_affected_cpus - show the CPUs affected by each transition
 759  */
 760 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 761 {
 762         return cpufreq_show_cpus(policy->cpus, buf);
 763 }
 764
 765 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 766                                         const char *buf, size_t count)
 767 {
 768         unsigned int freq = 0;
 769         unsigned int ret;
 770
 771         if (!policy->governor || !policy->governor->store_setspeed)
 772                 return -EINVAL;
 773
 774         ret = sscanf(buf, "%u", &freq);
 775         if (ret != 1)
 776                 return -EINVAL;
 777
 778         policy->governor->store_setspeed(policy, freq);
 779
 780         return count;
 781 }
 782
 783 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 784 {
 785         if (!policy->governor || !policy->governor->show_setspeed)
 786                 return sprintf(buf, "<unsupported>\n");
 787
 788         return policy->governor->show_setspeed(policy, buf);
 789 }
 790
 791 /**
 792  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 793  */
 794 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 795 {
 796         unsigned int limit;
 797         int ret;
 798         if (cpufreq_driver->bios_limit) {
 799                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 800                 if (!ret)
 801                         return sprintf(buf, "%u\n", limit);
 802         }
 803         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 804 }
 805
 806 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 807 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 808 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 809 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 810 cpufreq_freq_attr_ro(scaling_available_governors);
 811 cpufreq_freq_attr_ro(scaling_driver);
 812 cpufreq_freq_attr_ro(scaling_cur_freq);
 813 cpufreq_freq_attr_ro(bios_limit);
 814 cpufreq_freq_attr_ro(related_cpus);
 815 cpufreq_freq_attr_ro(affected_cpus);
 816 cpufreq_freq_attr_rw(scaling_min_freq);
 817 cpufreq_freq_attr_rw(scaling_max_freq);
 818 cpufreq_freq_attr_rw(scaling_governor);
 819 cpufreq_freq_attr_rw(scaling_setspeed);
 820
 821 static struct attribute *default_attrs[] = {
 822         &cpuinfo_min_freq.attr,
 823         &cpuinfo_max_freq.attr,
 824         &cpuinfo_transition_latency.attr,
 825         &scaling_min_freq.attr,
 826         &scaling_max_freq.attr,
 827         &affected_cpus.attr,
 828         &related_cpus.attr,
 829         &scaling_governor.attr,
 830         &scaling_driver.attr,
 831         &scaling_available_governors.attr,
 832         &scaling_setspeed.attr,
 833         NULL
 834 };
 835
 836 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 837 #define to_attr(a) container_of(a, struct freq_attr, attr)
 838
 839 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 840 {
 841         struct cpufreq_policy *policy = to_policy(kobj);
 842         struct freq_attr *fattr = to_attr(attr);
 843         ssize_t ret;
 844
 845         if (!down_read_trylock(&cpufreq_rwsem))
 846                 return -EINVAL;
 847
 848         down_read(&policy->rwsem);
 849
 850         if (fattr->show)
 851                 ret = fattr->show(policy, buf);
 852         else
 853                 ret = -EIO;
 854
 855         up_read(&policy->rwsem);
 856         up_read(&cpufreq_rwsem);
 857
 858         return ret;
 859 }
 860
 861 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 862                      const char *buf, size_t count)
 863 {
 864         struct cpufreq_policy *policy = to_policy(kobj);
 865         struct freq_attr *fattr = to_attr(attr);
 866         ssize_t ret = -EINVAL;
 867
 868         get_online_cpus();
 869
 870         if (!cpu_online(policy->cpu))
 871                 goto unlock;
 872
 873         if (!down_read_trylock(&cpufreq_rwsem))
 874                 goto unlock;
 875
 876         down_write(&policy->rwsem);
 877
 878         /* Updating inactive policies is invalid, so avoid doing that. */
 879         if (unlikely(policy_is_inactive(policy))) {
 880                 ret = -EBUSY;
 881                 goto unlock_policy_rwsem;
 882         }
 883
 884         if (fattr->store)
 885                 ret = fattr->store(policy, buf, count);
 886         else
 887                 ret = -EIO;
 888
 889 unlock_policy_rwsem:
 890         up_write(&policy->rwsem);
 891
 892         up_read(&cpufreq_rwsem);
 893 unlock:
 894         put_online_cpus();
 895
 896         return ret;
 897 }
 898
 899 static void cpufreq_sysfs_release(struct kobject *kobj)
 900 {
 901         struct cpufreq_policy *policy = to_policy(kobj);
 902         pr_debug("last reference is dropped\n");
 903         complete(&policy->kobj_unregister);
 904 }
 905
 906 static const struct sysfs_ops sysfs_ops = {
 907         .show   = show,
 908         .store  = store,
 909 };
 910
 911 static struct kobj_type ktype_cpufreq = {
 912         .sysfs_ops      = &sysfs_ops,
 913         .default_attrs  = default_attrs,
 914         .release        = cpufreq_sysfs_release,
 915 };
 916
 917 struct kobject *cpufreq_global_kobject;
 918 EXPORT_SYMBOL(cpufreq_global_kobject);
 919
 920 static int cpufreq_global_kobject_usage;
 921
 922 int cpufreq_get_global_kobject(void)
 923 {
 924         if (!cpufreq_global_kobject_usage++)
 925                 return kobject_add(cpufreq_global_kobject,
 926                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 927
 928         return 0;
 929 }
 930 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 931
 932 void cpufreq_put_global_kobject(void)
 933 {
 934         if (!--cpufreq_global_kobject_usage)
 935                 kobject_del(cpufreq_global_kobject);
 936 }
 937 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 938
 939 int cpufreq_sysfs_create_file(const struct attribute *attr)
 940 {
 941         int ret = cpufreq_get_global_kobject();
 942
 943         if (!ret) {
 944                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 945                 if (ret)
 946                         cpufreq_put_global_kobject();
 947         }
 948
 949         return ret;
 950 }
 951 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 952
 953 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 954 {
 955         sysfs_remove_file(cpufreq_global_kobject, attr);
 956         cpufreq_put_global_kobject();
 957 }
 958 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 959
 960 static int add_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 961 {
 962         struct device *cpu_dev;
 963
 964         pr_debug("%s: Adding symlink for CPU: %u\n", __func__, cpu);
 965
 966         if (!policy)
 967                 return 0;
 968
 969         cpu_dev = get_cpu_device(cpu);
 970         if (WARN_ON(!cpu_dev))
 971                 return 0;
 972
 973         return sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq");
 974 }
 975
 976 static void remove_cpu_dev_symlink(struct cpufreq_policy *policy, int cpu)
 977 {
 978         struct device *cpu_dev;
 979
 980         pr_debug("%s: Removing symlink for CPU: %u\n", __func__, cpu);
 981
 982         cpu_dev = get_cpu_device(cpu);
 983         if (WARN_ON(!cpu_dev))
 984                 return;
 985
 986         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
 987 }
 988
 989 /* Add/remove symlinks for all related CPUs */
 990 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 991 {
 992         unsigned int j;
 993         int ret = 0;
 994
 995         /* Some related CPUs might not be present (physically hotplugged) */
 996         for_each_cpu_and(j, policy->related_cpus, cpu_present_mask) {
 997                 if (j == policy->kobj_cpu)
 998                         continue;
 999
1000                 ret = add_cpu_dev_symlink(policy, j);
1001                 if (ret)
1002                         break;
1003         }
1004
1005         return ret;
1006 }
1007
1008 static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy)
1009 {
1010         unsigned int j;
1011
1012         /* Some related CPUs might not be present (physically hotplugged) */
1013         for_each_cpu_and(j, policy->related_cpus, cpu_present_mask) {
1014                 if (j == policy->kobj_cpu)
1015                         continue;
1016
1017                 remove_cpu_dev_symlink(policy, j);
1018         }
1019 }
1020
1021 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
1022                                      struct device *dev)
1023 {
1024         struct freq_attr **drv_attr;
1025         int ret = 0;
1026
1027         /* set up files for this cpu device */
1028         drv_attr = cpufreq_driver->attr;
1029         while (drv_attr && *drv_attr) {
1030                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1031                 if (ret)
1032                         return ret;
1033                 drv_attr++;
1034         }
1035         if (cpufreq_driver->get) {
1036                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1037                 if (ret)
1038                         return ret;
1039         }
1040
1041         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1042         if (ret)
1043                 return ret;
1044
1045         if (cpufreq_driver->bios_limit) {
1046                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1047                 if (ret)
1048                         return ret;
1049         }
1050
1051         return cpufreq_add_dev_symlink(policy);
1052 }
1053
1054 static void cpufreq_init_policy(struct cpufreq_policy *policy)
1055 {
1056         struct cpufreq_governor *gov = NULL;
1057         struct cpufreq_policy new_policy;
1058         int ret = 0;
1059
1060         memcpy(&new_policy, policy, sizeof(*policy));
1061
1062         /* Update governor of new_policy to the governor used before hotplug */
1063         gov = find_governor(policy->last_governor);
1064         if (gov)
1065                 pr_debug("Restoring governor %s for cpu %d\n",
1066                                 policy->governor->name, policy->cpu);
1067         else
1068                 gov = CPUFREQ_DEFAULT_GOVERNOR;
1069
1070         new_policy.governor = gov;
1071
1072         /* Use the default policy if its valid. */
1073         if (cpufreq_driver->setpolicy)
1074                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
1075
1076         /* set default policy */
1077         ret = cpufreq_set_policy(policy, &new_policy);
1078         if (ret) {
1079                 pr_debug("setting policy failed\n");
1080                 if (cpufreq_driver->exit)
1081                         cpufreq_driver->exit(policy);
1082         }
1083 }
1084
1085 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
1086                                   unsigned int cpu, struct device *dev)
1087 {
1088         int ret = 0;
1089
1090         /* Has this CPU been taken care of already? */
1091         if (cpumask_test_cpu(cpu, policy->cpus))
1092                 return 0;
1093
1094         if (has_target()) {
1095                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1096                 if (ret) {
1097                         pr_err("%s: Failed to stop governor\n", __func__);
1098                         return ret;
1099                 }
1100         }
1101
1102         down_write(&policy->rwsem);
1103         cpumask_set_cpu(cpu, policy->cpus);
1104         up_write(&policy->rwsem);
1105
1106         if (has_target()) {
1107                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1108                 if (!ret)
1109                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1110
1111                 if (ret) {
1112                         pr_err("%s: Failed to start governor\n", __func__);
1113                         return ret;
1114                 }
1115         }
1116
1117         return 0;
1118 }
1119
1120 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1121 {
1122         struct cpufreq_policy *policy;
1123         unsigned long flags;
1124
1125         read_lock_irqsave(&cpufreq_driver_lock, flags);
1126         policy = per_cpu(cpufreq_cpu_data, cpu);
1127         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1128
1129         if (likely(policy)) {
1130                 /* Policy should be inactive here */
1131                 WARN_ON(!policy_is_inactive(policy));
1132
1133                 down_write(&policy->rwsem);
1134                 policy->cpu = cpu;
1135                 policy->governor = NULL;
1136                 up_write(&policy->rwsem);
1137         }
1138
1139         return policy;
1140 }
1141
1142 static struct cpufreq_policy *cpufreq_policy_alloc(struct device *dev)
1143 {
1144         struct cpufreq_policy *policy;
1145         int ret;
1146
1147         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1148         if (!policy)
1149                 return NULL;
1150
1151         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1152                 goto err_free_policy;
1153
1154         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1155                 goto err_free_cpumask;
1156
1157         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &dev->kobj,
1158                                    "cpufreq");
1159         if (ret) {
1160                 pr_err("%s: failed to init policy->kobj: %d\n", __func__, ret);
1161                 goto err_free_rcpumask;
1162         }
1163
1164         INIT_LIST_HEAD(&policy->policy_list);
1165         init_rwsem(&policy->rwsem);
1166         spin_lock_init(&policy->transition_lock);
1167         init_waitqueue_head(&policy->transition_wait);
1168         init_completion(&policy->kobj_unregister);
1169         INIT_WORK(&policy->update, handle_update);
1170
1171         policy->cpu = dev->id;
1172
1173         /* Set this once on allocation */
1174         policy->kobj_cpu = dev->id;
1175
1176         return policy;
1177
1178 err_free_rcpumask:
1179         free_cpumask_var(policy->related_cpus);
1180 err_free_cpumask:
1181         free_cpumask_var(policy->cpus);
1182 err_free_policy:
1183         kfree(policy);
1184
1185         return NULL;
1186 }
1187
1188 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy, bool notify)
1189 {
1190         struct kobject *kobj;
1191         struct completion *cmp;
1192
1193         if (notify)
1194                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1195                                              CPUFREQ_REMOVE_POLICY, policy);
1196
1197         down_write(&policy->rwsem);
1198         cpufreq_remove_dev_symlink(policy);
1199         kobj = &policy->kobj;
1200         cmp = &policy->kobj_unregister;
1201         up_write(&policy->rwsem);
1202         kobject_put(kobj);
1203
1204         /*
1205          * We need to make sure that the underlying kobj is
1206          * actually not referenced anymore by anybody before we
1207          * proceed with unloading.
1208          */
1209         pr_debug("waiting for dropping of refcount\n");
1210         wait_for_completion(cmp);
1211         pr_debug("wait complete\n");
1212 }
1213
1214 static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify)
1215 {
1216         unsigned long flags;
1217         int cpu;
1218
1219         /* Remove policy from list */
1220         write_lock_irqsave(&cpufreq_driver_lock, flags);
1221         list_del(&policy->policy_list);
1222
1223         for_each_cpu(cpu, policy->related_cpus)
1224                 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1225         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1226
1227         cpufreq_policy_put_kobj(policy, notify);
1228         free_cpumask_var(policy->related_cpus);
1229         free_cpumask_var(policy->cpus);
1230         kfree(policy);
1231 }
1232
1233 /**
1234  * cpufreq_add_dev - add a CPU device
1235  *
1236  * Adds the cpufreq interface for a CPU device.
1237  *
1238  * The Oracle says: try running cpufreq registration/unregistration concurrently
1239  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1240  * mess up, but more thorough testing is needed. - Mathieu
1241  */
1242 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1243 {
1244         unsigned int j, cpu = dev->id;
1245         int ret = -ENOMEM;
1246         struct cpufreq_policy *policy;
1247         unsigned long flags;
1248         bool recover_policy = !sif;
1249
1250         pr_debug("adding CPU %u\n", cpu);
1251
1252         /*
1253          * Only possible if 'cpu' wasn't physically present earlier and we are
1254          * here from subsys_interface add callback. A hotplug notifier will
1255          * follow and we will handle it like logical CPU hotplug then. For now,
1256          * just create the sysfs link.
1257          */
1258         if (cpu_is_offline(cpu))
1259                 return add_cpu_dev_symlink(per_cpu(cpufreq_cpu_data, cpu), cpu);
1260
1261         if (!down_read_trylock(&cpufreq_rwsem))
1262                 return 0;
1263
1264         /* Check if this CPU already has a policy to manage it */
1265         policy = per_cpu(cpufreq_cpu_data, cpu);
1266         if (policy && !policy_is_inactive(policy)) {
1267                 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
1268                 ret = cpufreq_add_policy_cpu(policy, cpu, dev);
1269                 up_read(&cpufreq_rwsem);
1270                 return ret;
1271         }
1272
1273         /*
1274          * Restore the saved policy when doing light-weight init and fall back
1275          * to the full init if that fails.
1276          */
1277         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1278         if (!policy) {
1279                 recover_policy = false;
1280                 policy = cpufreq_policy_alloc(dev);
1281                 if (!policy)
1282                         goto nomem_out;
1283         }
1284
1285         cpumask_copy(policy->cpus, cpumask_of(cpu));
1286
1287         /* call driver. From then on the cpufreq must be able
1288          * to accept all calls to ->verify and ->setpolicy for this CPU
1289          */
1290         ret = cpufreq_driver->init(policy);
1291         if (ret) {
1292                 pr_debug("initialization failed\n");
1293                 goto err_set_policy_cpu;
1294         }
1295
1296         down_write(&policy->rwsem);
1297
1298         /* related cpus should atleast have policy->cpus */
1299         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1300
1301         /*
1302          * affected cpus must always be the one, which are online. We aren't
1303          * managing offline cpus here.
1304          */
1305         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1306
1307         if (!recover_policy) {
1308                 policy->user_policy.min = policy->min;
1309                 policy->user_policy.max = policy->max;
1310
1311                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1312                 for_each_cpu(j, policy->related_cpus)
1313                         per_cpu(cpufreq_cpu_data, j) = policy;
1314                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1315         }
1316
1317         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1318                 policy->cur = cpufreq_driver->get(policy->cpu);
1319                 if (!policy->cur) {
1320                         pr_err("%s: ->get() failed\n", __func__);
1321                         goto err_get_freq;
1322                 }
1323         }
1324
1325         /*
1326          * Sometimes boot loaders set CPU frequency to a value outside of
1327          * frequency table present with cpufreq core. In such cases CPU might be
1328          * unstable if it has to run on that frequency for long duration of time
1329          * and so its better to set it to a frequency which is specified in
1330          * freq-table. This also makes cpufreq stats inconsistent as
1331          * cpufreq-stats would fail to register because current frequency of CPU
1332          * isn't found in freq-table.
1333          *
1334          * Because we don't want this change to effect boot process badly, we go
1335          * for the next freq which is >= policy->cur ('cur' must be set by now,
1336          * otherwise we will end up setting freq to lowest of the table as 'cur'
1337          * is initialized to zero).
1338          *
1339          * We are passing target-freq as "policy->cur - 1" otherwise
1340          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1341          * equal to target-freq.
1342          */
1343         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1344             && has_target()) {
1345                 /* Are we running at unknown frequency ? */
1346                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1347                 if (ret == -EINVAL) {
1348                         /* Warn user and fix it */
1349                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1350                                 __func__, policy->cpu, policy->cur);
1351                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1352                                 CPUFREQ_RELATION_L);
1353
1354                         /*
1355                          * Reaching here after boot in a few seconds may not
1356                          * mean that system will remain stable at "unknown"
1357                          * frequency for longer duration. Hence, a BUG_ON().
1358                          */
1359                         BUG_ON(ret);
1360                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1361                                 __func__, policy->cpu, policy->cur);
1362                 }
1363         }
1364
1365         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1366                                      CPUFREQ_START, policy);
1367
1368         if (!recover_policy) {
1369                 ret = cpufreq_add_dev_interface(policy, dev);
1370                 if (ret)
1371                         goto err_out_unregister;
1372                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1373                                 CPUFREQ_CREATE_POLICY, policy);
1374
1375                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1376                 list_add(&policy->policy_list, &cpufreq_policy_list);
1377                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1378         }
1379
1380         cpufreq_init_policy(policy);
1381
1382         if (!recover_policy) {
1383                 policy->user_policy.policy = policy->policy;
1384                 policy->user_policy.governor = policy->governor;
1385         }
1386         up_write(&policy->rwsem);
1387
1388         kobject_uevent(&policy->kobj, KOBJ_ADD);
1389
1390         up_read(&cpufreq_rwsem);
1391
1392         /* Callback for handling stuff after policy is ready */
1393         if (cpufreq_driver->ready)
1394                 cpufreq_driver->ready(policy);
1395
1396         pr_debug("initialization complete\n");
1397
1398         return 0;
1399
1400 err_out_unregister:
1401 err_get_freq:
1402         up_write(&policy->rwsem);
1403
1404         if (cpufreq_driver->exit)
1405                 cpufreq_driver->exit(policy);
1406 err_set_policy_cpu:
1407         cpufreq_policy_free(policy, recover_policy);
1408 nomem_out:
1409         up_read(&cpufreq_rwsem);
1410
1411         return ret;
1412 }
1413
1414 static int __cpufreq_remove_dev_prepare(struct device *dev,
1415                                         struct subsys_interface *sif)
1416 {
1417         unsigned int cpu = dev->id;
1418         int ret = 0;
1419         struct cpufreq_policy *policy;
1420
1421         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1422
1423         policy = cpufreq_cpu_get_raw(cpu);
1424         if (!policy) {
1425                 pr_debug("%s: No cpu_data found\n", __func__);
1426                 return -EINVAL;
1427         }
1428
1429         if (has_target()) {
1430                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1431                 if (ret) {
1432                         pr_err("%s: Failed to stop governor\n", __func__);
1433                         return ret;
1434                 }
1435         }
1436
1437         down_write(&policy->rwsem);
1438         cpumask_clear_cpu(cpu, policy->cpus);
1439
1440         if (policy_is_inactive(policy)) {
1441                 if (has_target())
1442                         strncpy(policy->last_governor, policy->governor->name,
1443                                 CPUFREQ_NAME_LEN);
1444         } else if (cpu == policy->cpu) {
1445                 /* Nominate new CPU */
1446                 policy->cpu = cpumask_any(policy->cpus);
1447         }
1448         up_write(&policy->rwsem);
1449
1450         /* Start governor again for active policy */
1451         if (!policy_is_inactive(policy)) {
1452                 if (has_target()) {
1453                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1454                         if (!ret)
1455                                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1456
1457                         if (ret)
1458                                 pr_err("%s: Failed to start governor\n", __func__);
1459                 }
1460         } else if (cpufreq_driver->stop_cpu) {
1461                 cpufreq_driver->stop_cpu(policy);
1462         }
1463
1464         return ret;
1465 }
1466
1467 static int __cpufreq_remove_dev_finish(struct device *dev,
1468                                        struct subsys_interface *sif)
1469 {
1470         unsigned int cpu = dev->id;
1471         int ret;
1472         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1473
1474         if (!policy) {
1475                 pr_debug("%s: No cpu_data found\n", __func__);
1476                 return -EINVAL;
1477         }
1478
1479         /* Only proceed for inactive policies */
1480         if (!policy_is_inactive(policy))
1481                 return 0;
1482
1483         /* If cpu is last user of policy, free policy */
1484         if (has_target()) {
1485                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
1486                 if (ret) {
1487                         pr_err("%s: Failed to exit governor\n", __func__);
1488                         return ret;
1489                 }
1490         }
1491
1492         /*
1493          * Perform the ->exit() even during light-weight tear-down,
1494          * since this is a core component, and is essential for the
1495          * subsequent light-weight ->init() to succeed.
1496          */
1497         if (cpufreq_driver->exit)
1498                 cpufreq_driver->exit(policy);
1499
1500         /* Free the policy only if the driver is getting removed. */
1501         if (sif)
1502                 cpufreq_policy_free(policy, true);
1503
1504         return 0;
1505 }
1506
1507 /**
1508  * cpufreq_remove_dev - remove a CPU device
1509  *
1510  * Removes the cpufreq interface for a CPU device.
1511  */
1512 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1513 {
1514         unsigned int cpu = dev->id;
1515         int ret;
1516
1517         /*
1518          * Only possible if 'cpu' is getting physically removed now. A hotplug
1519          * notifier should have already been called and we just need to remove
1520          * link or free policy here.
1521          */
1522         if (cpu_is_offline(cpu)) {
1523                 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1524                 struct cpumask mask;
1525
1526                 if (!policy)
1527                         return 0;
1528
1529                 cpumask_copy(&mask, policy->related_cpus);
1530                 cpumask_clear_cpu(cpu, &mask);
1531
1532                 /*
1533                  * Free policy only if all policy->related_cpus are removed
1534                  * physically.
1535                  */
1536                 if (cpumask_intersects(&mask, cpu_present_mask)) {
1537                         remove_cpu_dev_symlink(policy, cpu);
1538                         return 0;
1539                 }
1540
1541                 cpufreq_policy_free(policy, true);
1542                 return 0;
1543         }
1544
1545         ret = __cpufreq_remove_dev_prepare(dev, sif);
1546
1547         if (!ret)
1548                 ret = __cpufreq_remove_dev_finish(dev, sif);
1549
1550         return ret;
1551 }
1552
1553 static void handle_update(struct work_struct *work)
1554 {
1555         struct cpufreq_policy *policy =
1556                 container_of(work, struct cpufreq_policy, update);
1557         unsigned int cpu = policy->cpu;
1558         pr_debug("handle_update for cpu %u called\n", cpu);
1559         cpufreq_update_policy(cpu);
1560 }
1561
1562 /**
1563  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1564  *      in deep trouble.
1565  *      @policy: policy managing CPUs
1566  *      @new_freq: CPU frequency the CPU actually runs at
1567  *
1568  *      We adjust to current frequency first, and need to clean up later.
1569  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1570  */
1571 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1572                                 unsigned int new_freq)
1573 {
1574         struct cpufreq_freqs freqs;
1575
1576         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1577                  policy->cur, new_freq);
1578
1579         freqs.old = policy->cur;
1580         freqs.new = new_freq;
1581
1582         cpufreq_freq_transition_begin(policy, &freqs);
1583         cpufreq_freq_transition_end(policy, &freqs, 0);
1584 }
1585
1586 /**
1587  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1588  * @cpu: CPU number
1589  *
1590  * This is the last known freq, without actually getting it from the driver.
1591  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1592  */
1593 unsigned int cpufreq_quick_get(unsigned int cpu)
1594 {
1595         struct cpufreq_policy *policy;
1596         unsigned int ret_freq = 0;
1597
1598         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1599                 return cpufreq_driver->get(cpu);
1600
1601         policy = cpufreq_cpu_get(cpu);
1602         if (policy) {
1603                 ret_freq = policy->cur;
1604                 cpufreq_cpu_put(policy);
1605         }
1606
1607         return ret_freq;
1608 }
1609 EXPORT_SYMBOL(cpufreq_quick_get);
1610
1611 /**
1612  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1613  * @cpu: CPU number
1614  *
1615  * Just return the max possible frequency for a given CPU.
1616  */
1617 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1618 {
1619         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1620         unsigned int ret_freq = 0;
1621
1622         if (policy) {
1623                 ret_freq = policy->max;
1624                 cpufreq_cpu_put(policy);
1625         }
1626
1627         return ret_freq;
1628 }
1629 EXPORT_SYMBOL(cpufreq_quick_get_max);
1630
1631 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1632 {
1633         unsigned int ret_freq = 0;
1634
1635         if (!cpufreq_driver->get)
1636                 return ret_freq;
1637
1638         ret_freq = cpufreq_driver->get(policy->cpu);
1639
1640         /* Updating inactive policies is invalid, so avoid doing that. */
1641         if (unlikely(policy_is_inactive(policy)))
1642                 return ret_freq;
1643
1644         if (ret_freq && policy->cur &&
1645                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1646                 /* verify no discrepancy between actual and
1647                                         saved value exists */
1648                 if (unlikely(ret_freq != policy->cur)) {
1649                         cpufreq_out_of_sync(policy, ret_freq);
1650                         schedule_work(&policy->update);
1651                 }
1652         }
1653
1654         return ret_freq;
1655 }
1656
1657 /**
1658  * cpufreq_get - get the current CPU frequency (in kHz)
1659  * @cpu: CPU number
1660  *
1661  * Get the CPU current (static) CPU frequency
1662  */
1663 unsigned int cpufreq_get(unsigned int cpu)
1664 {
1665         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1666         unsigned int ret_freq = 0;
1667
1668         if (policy) {
1669                 down_read(&policy->rwsem);
1670                 ret_freq = __cpufreq_get(policy);
1671                 up_read(&policy->rwsem);
1672
1673                 cpufreq_cpu_put(policy);
1674         }
1675
1676         return ret_freq;
1677 }
1678 EXPORT_SYMBOL(cpufreq_get);
1679
1680 static struct subsys_interface cpufreq_interface = {
1681         .name           = "cpufreq",
1682         .subsys         = &cpu_subsys,
1683         .add_dev        = cpufreq_add_dev,
1684         .remove_dev     = cpufreq_remove_dev,
1685 };
1686
1687 /*
1688  * In case platform wants some specific frequency to be configured
1689  * during suspend..
1690  */
1691 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1692 {
1693         int ret;
1694
1695         if (!policy->suspend_freq) {
1696                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1697                 return -EINVAL;
1698         }
1699
1700         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1701                         policy->suspend_freq);
1702
1703         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1704                         CPUFREQ_RELATION_H);
1705         if (ret)
1706                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1707                                 __func__, policy->suspend_freq, ret);
1708
1709         return ret;
1710 }
1711 EXPORT_SYMBOL(cpufreq_generic_suspend);
1712
1713 /**
1714  * cpufreq_suspend() - Suspend CPUFreq governors
1715  *
1716  * Called during system wide Suspend/Hibernate cycles for suspending governors
1717  * as some platforms can't change frequency after this point in suspend cycle.
1718  * Because some of the devices (like: i2c, regulators, etc) they use for
1719  * changing frequency are suspended quickly after this point.
1720  */
1721 void cpufreq_suspend(void)
1722 {
1723         struct cpufreq_policy *policy;
1724
1725         if (!cpufreq_driver)
1726                 return;
1727
1728         if (!has_target())
1729                 goto suspend;
1730
1731         pr_debug("%s: Suspending Governors\n", __func__);
1732
1733         for_each_active_policy(policy) {
1734                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1735                         pr_err("%s: Failed to stop governor for policy: %p\n",
1736                                 __func__, policy);
1737                 else if (cpufreq_driver->suspend
1738                     && cpufreq_driver->suspend(policy))
1739                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1740                                 policy);
1741         }
1742
1743 suspend:
1744         cpufreq_suspended = true;
1745 }
1746
1747 /**
1748  * cpufreq_resume() - Resume CPUFreq governors
1749  *
1750  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1751  * are suspended with cpufreq_suspend().
1752  */
1753 void cpufreq_resume(void)
1754 {
1755         struct cpufreq_policy *policy;
1756
1757         if (!cpufreq_driver)
1758                 return;
1759
1760         cpufreq_suspended = false;
1761
1762         if (!has_target())
1763                 return;
1764
1765         pr_debug("%s: Resuming Governors\n", __func__);
1766
1767         for_each_active_policy(policy) {
1768                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1769                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1770                                 policy);
1771                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1772                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1773                         pr_err("%s: Failed to start governor for policy: %p\n",
1774                                 __func__, policy);
1775         }
1776
1777         /*
1778          * schedule call cpufreq_update_policy() for first-online CPU, as that
1779          * wouldn't be hotplugged-out on suspend. It will verify that the
1780          * current freq is in sync with what we believe it to be.
1781          */
1782         policy = cpufreq_cpu_get_raw(cpumask_first(cpu_online_mask));
1783         if (WARN_ON(!policy))
1784                 return;
1785
1786         schedule_work(&policy->update);
1787 }
1788
1789 /**
1790  *      cpufreq_get_current_driver - return current driver's name
1791  *
1792  *      Return the name string of the currently loaded cpufreq driver
1793  *      or NULL, if none.
1794  */
1795 const char *cpufreq_get_current_driver(void)
1796 {
1797         if (cpufreq_driver)
1798                 return cpufreq_driver->name;
1799
1800         return NULL;
1801 }
1802 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1803
1804 /**
1805  *      cpufreq_get_driver_data - return current driver data
1806  *
1807  *      Return the private data of the currently loaded cpufreq
1808  *      driver, or NULL if no cpufreq driver is loaded.
1809  */
1810 void *cpufreq_get_driver_data(void)
1811 {
1812         if (cpufreq_driver)
1813                 return cpufreq_driver->driver_data;
1814
1815         return NULL;
1816 }
1817 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1818
1819 /*********************************************************************
1820  *                     NOTIFIER LISTS INTERFACE                      *
1821  *********************************************************************/
1822
1823 /**
1824  *      cpufreq_register_notifier - register a driver with cpufreq
1825  *      @nb: notifier function to register
1826  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1827  *
1828  *      Add a driver to one of two lists: either a list of drivers that
1829  *      are notified about clock rate changes (once before and once after
1830  *      the transition), or a list of drivers that are notified about
1831  *      changes in cpufreq policy.
1832  *
1833  *      This function may sleep, and has the same return conditions as
1834  *      blocking_notifier_chain_register.
1835  */
1836 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1837 {
1838         int ret;
1839
1840         if (cpufreq_disabled())
1841                 return -EINVAL;
1842
1843         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1844
1845         switch (list) {
1846         case CPUFREQ_TRANSITION_NOTIFIER:
1847                 ret = srcu_notifier_chain_register(
1848                                 &cpufreq_transition_notifier_list, nb);
1849                 break;
1850         case CPUFREQ_POLICY_NOTIFIER:
1851                 ret = blocking_notifier_chain_register(
1852                                 &cpufreq_policy_notifier_list, nb);
1853                 break;
1854         default:
1855                 ret = -EINVAL;
1856         }
1857
1858         return ret;
1859 }
1860 EXPORT_SYMBOL(cpufreq_register_notifier);
1861
1862 /**
1863  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1864  *      @nb: notifier block to be unregistered
1865  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1866  *
1867  *      Remove a driver from the CPU frequency notifier list.
1868  *
1869  *      This function may sleep, and has the same return conditions as
1870  *      blocking_notifier_chain_unregister.
1871  */
1872 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1873 {
1874         int ret;
1875
1876         if (cpufreq_disabled())
1877                 return -EINVAL;
1878
1879         switch (list) {
1880         case CPUFREQ_TRANSITION_NOTIFIER:
1881                 ret = srcu_notifier_chain_unregister(
1882                                 &cpufreq_transition_notifier_list, nb);
1883                 break;
1884         case CPUFREQ_POLICY_NOTIFIER:
1885                 ret = blocking_notifier_chain_unregister(
1886                                 &cpufreq_policy_notifier_list, nb);
1887                 break;
1888         default:
1889                 ret = -EINVAL;
1890         }
1891
1892         return ret;
1893 }
1894 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1895
1896
1897 /*********************************************************************
1898  *                              GOVERNORS                            *
1899  *********************************************************************/
1900
1901 /* Must set freqs->new to intermediate frequency */
1902 static int __target_intermediate(struct cpufreq_policy *policy,
1903                                  struct cpufreq_freqs *freqs, int index)
1904 {
1905         int ret;
1906
1907         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1908
1909         /* We don't need to switch to intermediate freq */
1910         if (!freqs->new)
1911                 return 0;
1912
1913         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1914                  __func__, policy->cpu, freqs->old, freqs->new);
1915
1916         cpufreq_freq_transition_begin(policy, freqs);
1917         ret = cpufreq_driver->target_intermediate(policy, index);
1918         cpufreq_freq_transition_end(policy, freqs, ret);
1919
1920         if (ret)
1921                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1922                        __func__, ret);
1923
1924         return ret;
1925 }
1926
1927 static int __target_index(struct cpufreq_policy *policy,
1928                           struct cpufreq_frequency_table *freq_table, int index)
1929 {
1930         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1931         unsigned int intermediate_freq = 0;
1932         int retval = -EINVAL;
1933         bool notify;
1934
1935         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1936         if (notify) {
1937                 /* Handle switching to intermediate frequency */
1938                 if (cpufreq_driver->get_intermediate) {
1939                         retval = __target_intermediate(policy, &freqs, index);
1940                         if (retval)
1941                                 return retval;
1942
1943                         intermediate_freq = freqs.new;
1944                         /* Set old freq to intermediate */
1945                         if (intermediate_freq)
1946                                 freqs.old = freqs.new;
1947                 }
1948
1949                 freqs.new = freq_table[index].frequency;
1950                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1951                          __func__, policy->cpu, freqs.old, freqs.new);
1952
1953                 cpufreq_freq_transition_begin(policy, &freqs);
1954         }
1955
1956         retval = cpufreq_driver->target_index(policy, index);
1957         if (retval)
1958                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1959                        retval);
1960
1961         if (notify) {
1962                 cpufreq_freq_transition_end(policy, &freqs, retval);
1963
1964                 /*
1965                  * Failed after setting to intermediate freq? Driver should have
1966                  * reverted back to initial frequency and so should we. Check
1967                  * here for intermediate_freq instead of get_intermediate, in
1968                  * case we haven't switched to intermediate freq at all.
1969                  */
1970                 if (unlikely(retval && intermediate_freq)) {
1971                         freqs.old = intermediate_freq;
1972                         freqs.new = policy->restore_freq;
1973                         cpufreq_freq_transition_begin(policy, &freqs);
1974                         cpufreq_freq_transition_end(policy, &freqs, 0);
1975                 }
1976         }
1977
1978         return retval;
1979 }
1980
1981 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1982                             unsigned int target_freq,
1983                             unsigned int relation)
1984 {
1985         unsigned int old_target_freq = target_freq;
1986         int retval = -EINVAL;
1987
1988         if (cpufreq_disabled())
1989                 return -ENODEV;
1990
1991         /* Make sure that target_freq is within supported range */
1992         if (target_freq > policy->max)
1993                 target_freq = policy->max;
1994         if (target_freq < policy->min)
1995                 target_freq = policy->min;
1996
1997         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1998                  policy->cpu, target_freq, relation, old_target_freq);
1999
2000         /*
2001          * This might look like a redundant call as we are checking it again
2002          * after finding index. But it is left intentionally for cases where
2003          * exactly same freq is called again and so we can save on few function
2004          * calls.
2005          */
2006         if (target_freq == policy->cur)
2007                 return 0;
2008
2009         /* Save last value to restore later on errors */
2010         policy->restore_freq = policy->cur;
2011
2012         if (cpufreq_driver->target)
2013                 retval = cpufreq_driver->target(policy, target_freq, relation);
2014         else if (cpufreq_driver->target_index) {
2015                 struct cpufreq_frequency_table *freq_table;
2016                 int index;
2017
2018                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2019                 if (unlikely(!freq_table)) {
2020                         pr_err("%s: Unable to find freq_table\n", __func__);
2021                         goto out;
2022                 }
2023
2024                 retval = cpufreq_frequency_table_target(policy, freq_table,
2025                                 target_freq, relation, &index);
2026                 if (unlikely(retval)) {
2027                         pr_err("%s: Unable to find matching freq\n", __func__);
2028                         goto out;
2029                 }
2030
2031                 if (freq_table[index].frequency == policy->cur) {
2032                         retval = 0;
2033                         goto out;
2034                 }
2035
2036                 retval = __target_index(policy, freq_table, index);
2037         }
2038
2039 out:
2040         return retval;
2041 }
2042 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2043
2044 int cpufreq_driver_target(struct cpufreq_policy *policy,
2045                           unsigned int target_freq,
2046                           unsigned int relation)
2047 {
2048         int ret = -EINVAL;
2049
2050         down_write(&policy->rwsem);
2051
2052         ret = __cpufreq_driver_target(policy, target_freq, relation);
2053
2054         up_write(&policy->rwsem);
2055
2056         return ret;
2057 }
2058 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2059
2060 static int __cpufreq_governor(struct cpufreq_policy *policy,
2061                                         unsigned int event)
2062 {
2063         int ret;
2064
2065         /* Only must be defined when default governor is known to have latency
2066            restrictions, like e.g. conservative or ondemand.
2067            That this is the case is already ensured in Kconfig
2068         */
2069 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2070         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2071 #else
2072         struct cpufreq_governor *gov = NULL;
2073 #endif
2074
2075         /* Don't start any governor operations if we are entering suspend */
2076         if (cpufreq_suspended)
2077                 return 0;
2078         /*
2079          * Governor might not be initiated here if ACPI _PPC changed
2080          * notification happened, so check it.
2081          */
2082         if (!policy->governor)
2083                 return -EINVAL;
2084
2085         if (policy->governor->max_transition_latency &&
2086             policy->cpuinfo.transition_latency >
2087             policy->governor->max_transition_latency) {
2088                 if (!gov)
2089                         return -EINVAL;
2090                 else {
2091                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2092                                 policy->governor->name, gov->name);
2093                         policy->governor = gov;
2094                 }
2095         }
2096
2097         if (event == CPUFREQ_GOV_POLICY_INIT)
2098                 if (!try_module_get(policy->governor->owner))
2099                         return -EINVAL;
2100
2101         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
2102                  policy->cpu, event);
2103
2104         mutex_lock(&cpufreq_governor_lock);
2105         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2106             || (!policy->governor_enabled
2107             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2108                 mutex_unlock(&cpufreq_governor_lock);
2109                 return -EBUSY;
2110         }
2111
2112         if (event == CPUFREQ_GOV_STOP)
2113                 policy->governor_enabled = false;
2114         else if (event == CPUFREQ_GOV_START)
2115                 policy->governor_enabled = true;
2116
2117         mutex_unlock(&cpufreq_governor_lock);
2118
2119         ret = policy->governor->governor(policy, event);
2120
2121         if (!ret) {
2122                 if (event == CPUFREQ_GOV_POLICY_INIT)
2123                         policy->governor->initialized++;
2124                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2125                         policy->governor->initialized--;
2126         } else {
2127                 /* Restore original values */
2128                 mutex_lock(&cpufreq_governor_lock);
2129                 if (event == CPUFREQ_GOV_STOP)
2130                         policy->governor_enabled = true;
2131                 else if (event == CPUFREQ_GOV_START)
2132                         policy->governor_enabled = false;
2133                 mutex_unlock(&cpufreq_governor_lock);
2134         }
2135
2136         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2137                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2138                 module_put(policy->governor->owner);
2139
2140         return ret;
2141 }
2142
2143 int cpufreq_register_governor(struct cpufreq_governor *governor)
2144 {
2145         int err;
2146
2147         if (!governor)
2148                 return -EINVAL;
2149
2150         if (cpufreq_disabled())
2151                 return -ENODEV;
2152
2153         mutex_lock(&cpufreq_governor_mutex);
2154
2155         governor->initialized = 0;
2156         err = -EBUSY;
2157         if (!find_governor(governor->name)) {
2158                 err = 0;
2159                 list_add(&governor->governor_list, &cpufreq_governor_list);
2160         }
2161
2162         mutex_unlock(&cpufreq_governor_mutex);
2163         return err;
2164 }
2165 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2166
2167 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2168 {
2169         struct cpufreq_policy *policy;
2170         unsigned long flags;
2171
2172         if (!governor)
2173                 return;
2174
2175         if (cpufreq_disabled())
2176                 return;
2177
2178         /* clear last_governor for all inactive policies */
2179         read_lock_irqsave(&cpufreq_driver_lock, flags);
2180         for_each_inactive_policy(policy) {
2181                 if (!strcmp(policy->last_governor, governor->name)) {
2182                         policy->governor = NULL;
2183                         strcpy(policy->last_governor, "\0");
2184                 }
2185         }
2186         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
2187
2188         mutex_lock(&cpufreq_governor_mutex);
2189         list_del(&governor->governor_list);
2190         mutex_unlock(&cpufreq_governor_mutex);
2191         return;
2192 }
2193 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2194
2195
2196 /*********************************************************************
2197  *                          POLICY INTERFACE                         *
2198  *********************************************************************/
2199
2200 /**
2201  * cpufreq_get_policy - get the current cpufreq_policy
2202  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2203  *      is written
2204  *
2205  * Reads the current cpufreq policy.
2206  */
2207 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2208 {
2209         struct cpufreq_policy *cpu_policy;
2210         if (!policy)
2211                 return -EINVAL;
2212
2213         cpu_policy = cpufreq_cpu_get(cpu);
2214         if (!cpu_policy)
2215                 return -EINVAL;
2216
2217         memcpy(policy, cpu_policy, sizeof(*policy));
2218
2219         cpufreq_cpu_put(cpu_policy);
2220         return 0;
2221 }
2222 EXPORT_SYMBOL(cpufreq_get_policy);
2223
2224 /*
2225  * policy : current policy.
2226  * new_policy: policy to be set.
2227  */
2228 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2229                                 struct cpufreq_policy *new_policy)
2230 {
2231         struct cpufreq_governor *old_gov;
2232         int ret;
2233
2234         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2235                  new_policy->cpu, new_policy->min, new_policy->max);
2236
2237         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2238
2239         if (new_policy->min > policy->max || new_policy->max < policy->min)
2240                 return -EINVAL;
2241
2242         /* verify the cpu speed can be set within this limit */
2243         ret = cpufreq_driver->verify(new_policy);
2244         if (ret)
2245                 return ret;
2246
2247         /* adjust if necessary - all reasons */
2248         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2249                         CPUFREQ_ADJUST, new_policy);
2250
2251         /* adjust if necessary - hardware incompatibility*/
2252         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2253                         CPUFREQ_INCOMPATIBLE, new_policy);
2254
2255         /*
2256          * verify the cpu speed can be set within this limit, which might be
2257          * different to the first one
2258          */
2259         ret = cpufreq_driver->verify(new_policy);
2260         if (ret)
2261                 return ret;
2262
2263         /* notification of the new policy */
2264         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2265                         CPUFREQ_NOTIFY, new_policy);
2266
2267         policy->min = new_policy->min;
2268         policy->max = new_policy->max;
2269
2270         pr_debug("new min and max freqs are %u - %u kHz\n",
2271                  policy->min, policy->max);
2272
2273         if (cpufreq_driver->setpolicy) {
2274                 policy->policy = new_policy->policy;
2275                 pr_debug("setting range\n");
2276                 return cpufreq_driver->setpolicy(new_policy);
2277         }
2278
2279         if (new_policy->governor == policy->governor)
2280                 goto out;
2281
2282         pr_debug("governor switch\n");
2283
2284         /* save old, working values */
2285         old_gov = policy->governor;
2286         /* end old governor */
2287         if (old_gov) {
2288                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2289                 up_write(&policy->rwsem);
2290                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2291                 down_write(&policy->rwsem);
2292         }
2293
2294         /* start new governor */
2295         policy->governor = new_policy->governor;
2296         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2297                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2298                         goto out;
2299
2300                 up_write(&policy->rwsem);
2301                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2302                 down_write(&policy->rwsem);
2303         }
2304
2305         /* new governor failed, so re-start old one */
2306         pr_debug("starting governor %s failed\n", policy->governor->name);
2307         if (old_gov) {
2308                 policy->governor = old_gov;
2309                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2310                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2311         }
2312
2313         return -EINVAL;
2314
2315  out:
2316         pr_debug("governor: change or update limits\n");
2317         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2318 }
2319
2320 /**
2321  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2322  *      @cpu: CPU which shall be re-evaluated
2323  *
2324  *      Useful for policy notifiers which have different necessities
2325  *      at different times.
2326  */
2327 int cpufreq_update_policy(unsigned int cpu)
2328 {
2329         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2330         struct cpufreq_policy new_policy;
2331         int ret;
2332
2333         if (!policy)
2334                 return -ENODEV;
2335
2336         down_write(&policy->rwsem);
2337
2338         pr_debug("updating policy for CPU %u\n", cpu);
2339         memcpy(&new_policy, policy, sizeof(*policy));
2340         new_policy.min = policy->user_policy.min;
2341         new_policy.max = policy->user_policy.max;
2342         new_policy.policy = policy->user_policy.policy;
2343         new_policy.governor = policy->user_policy.governor;
2344
2345         /*
2346          * BIOS might change freq behind our back
2347          * -> ask driver for current freq and notify governors about a change
2348          */
2349         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2350                 new_policy.cur = cpufreq_driver->get(cpu);
2351                 if (WARN_ON(!new_policy.cur)) {
2352                         ret = -EIO;
2353                         goto unlock;
2354                 }
2355
2356                 if (!policy->cur) {
2357                         pr_debug("Driver did not initialize current freq\n");
2358                         policy->cur = new_policy.cur;
2359                 } else {
2360                         if (policy->cur != new_policy.cur && has_target())
2361                                 cpufreq_out_of_sync(policy, new_policy.cur);
2362                 }
2363         }
2364
2365         ret = cpufreq_set_policy(policy, &new_policy);
2366
2367 unlock:
2368         up_write(&policy->rwsem);
2369
2370         cpufreq_cpu_put(policy);
2371         return ret;
2372 }
2373 EXPORT_SYMBOL(cpufreq_update_policy);
2374
2375 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2376                                         unsigned long action, void *hcpu)
2377 {
2378         unsigned int cpu = (unsigned long)hcpu;
2379         struct device *dev;
2380
2381         dev = get_cpu_device(cpu);
2382         if (dev) {
2383                 switch (action & ~CPU_TASKS_FROZEN) {
2384                 case CPU_ONLINE:
2385                         cpufreq_add_dev(dev, NULL);
2386                         break;
2387
2388                 case CPU_DOWN_PREPARE:
2389                         __cpufreq_remove_dev_prepare(dev, NULL);
2390                         break;
2391
2392                 case CPU_POST_DEAD:
2393                         __cpufreq_remove_dev_finish(dev, NULL);
2394                         break;
2395
2396                 case CPU_DOWN_FAILED:
2397                         cpufreq_add_dev(dev, NULL);
2398                         break;
2399                 }
2400         }
2401         return NOTIFY_OK;
2402 }
2403
2404 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2405         .notifier_call = cpufreq_cpu_callback,
2406 };
2407
2408 /*********************************************************************
2409  *               BOOST                                               *
2410  *********************************************************************/
2411 static int cpufreq_boost_set_sw(int state)
2412 {
2413         struct cpufreq_frequency_table *freq_table;
2414         struct cpufreq_policy *policy;
2415         int ret = -EINVAL;
2416
2417         for_each_active_policy(policy) {
2418                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2419                 if (freq_table) {
2420                         ret = cpufreq_frequency_table_cpuinfo(policy,
2421                                                         freq_table);
2422                         if (ret) {
2423                                 pr_err("%s: Policy frequency update failed\n",
2424                                        __func__);
2425                                 break;
2426                         }
2427                         policy->user_policy.max = policy->max;
2428                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2429                 }
2430         }
2431
2432         return ret;
2433 }
2434
2435 int cpufreq_boost_trigger_state(int state)
2436 {
2437         unsigned long flags;
2438         int ret = 0;
2439
2440         if (cpufreq_driver->boost_enabled == state)
2441                 return 0;
2442
2443         write_lock_irqsave(&cpufreq_driver_lock, flags);
2444         cpufreq_driver->boost_enabled = state;
2445         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2446
2447         ret = cpufreq_driver->set_boost(state);
2448         if (ret) {
2449                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2450                 cpufreq_driver->boost_enabled = !state;
2451                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2452
2453                 pr_err("%s: Cannot %s BOOST\n",
2454                        __func__, state ? "enable" : "disable");
2455         }
2456
2457         return ret;
2458 }
2459
2460 int cpufreq_boost_supported(void)
2461 {
2462         if (likely(cpufreq_driver))
2463                 return cpufreq_driver->boost_supported;
2464
2465         return 0;
2466 }
2467 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2468
2469 int cpufreq_boost_enabled(void)
2470 {
2471         return cpufreq_driver->boost_enabled;
2472 }
2473 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2474
2475 /*********************************************************************
2476  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2477  *********************************************************************/
2478
2479 /**
2480  * cpufreq_register_driver - register a CPU Frequency driver
2481  * @driver_data: A struct cpufreq_driver containing the values#
2482  * submitted by the CPU Frequency driver.
2483  *
2484  * Registers a CPU Frequency driver to this core code. This code
2485  * returns zero on success, -EBUSY when another driver got here first
2486  * (and isn't unregistered in the meantime).
2487  *
2488  */
2489 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2490 {
2491         unsigned long flags;
2492         int ret;
2493
2494         if (cpufreq_disabled())
2495                 return -ENODEV;
2496
2497         if (!driver_data || !driver_data->verify || !driver_data->init ||
2498             !(driver_data->setpolicy || driver_data->target_index ||
2499                     driver_data->target) ||
2500              (driver_data->setpolicy && (driver_data->target_index ||
2501                     driver_data->target)) ||
2502              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2503                 return -EINVAL;
2504
2505         pr_debug("trying to register driver %s\n", driver_data->name);
2506
2507         write_lock_irqsave(&cpufreq_driver_lock, flags);
2508         if (cpufreq_driver) {
2509                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2510                 return -EEXIST;
2511         }
2512         cpufreq_driver = driver_data;
2513         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2514
2515         if (driver_data->setpolicy)
2516                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2517
2518         if (cpufreq_boost_supported()) {
2519                 /*
2520                  * Check if driver provides function to enable boost -
2521                  * if not, use cpufreq_boost_set_sw as default
2522                  */
2523                 if (!cpufreq_driver->set_boost)
2524                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2525
2526                 ret = cpufreq_sysfs_create_file(&boost.attr);
2527                 if (ret) {
2528                         pr_err("%s: cannot register global BOOST sysfs file\n",
2529                                __func__);
2530                         goto err_null_driver;
2531                 }
2532         }
2533
2534         ret = subsys_interface_register(&cpufreq_interface);
2535         if (ret)
2536                 goto err_boost_unreg;
2537
2538         if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2539             list_empty(&cpufreq_policy_list)) {
2540                 /* if all ->init() calls failed, unregister */
2541                 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2542                          driver_data->name);
2543                 goto err_if_unreg;
2544         }
2545
2546         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2547         pr_debug("driver %s up and running\n", driver_data->name);
2548
2549         return 0;
2550 err_if_unreg:
2551         subsys_interface_unregister(&cpufreq_interface);
2552 err_boost_unreg:
2553         if (cpufreq_boost_supported())
2554                 cpufreq_sysfs_remove_file(&boost.attr);
2555 err_null_driver:
2556         write_lock_irqsave(&cpufreq_driver_lock, flags);
2557         cpufreq_driver = NULL;
2558         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2559         return ret;
2560 }
2561 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2562
2563 /**
2564  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2565  *
2566  * Unregister the current CPUFreq driver. Only call this if you have
2567  * the right to do so, i.e. if you have succeeded in initialising before!
2568  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2569  * currently not initialised.
2570  */
2571 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2572 {
2573         unsigned long flags;
2574
2575         if (!cpufreq_driver || (driver != cpufreq_driver))
2576                 return -EINVAL;
2577
2578         pr_debug("unregistering driver %s\n", driver->name);
2579
2580         subsys_interface_unregister(&cpufreq_interface);
2581         if (cpufreq_boost_supported())
2582                 cpufreq_sysfs_remove_file(&boost.attr);
2583
2584         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2585
2586         down_write(&cpufreq_rwsem);
2587         write_lock_irqsave(&cpufreq_driver_lock, flags);
2588
2589         cpufreq_driver = NULL;
2590
2591         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2592         up_write(&cpufreq_rwsem);
2593
2594         return 0;
2595 }
2596 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2597
2598 /*
2599  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2600  * or mutexes when secondary CPUs are halted.
2601  */
2602 static struct syscore_ops cpufreq_syscore_ops = {
2603         .shutdown = cpufreq_suspend,
2604 };
2605
2606 static int __init cpufreq_core_init(void)
2607 {
2608         if (cpufreq_disabled())
2609                 return -ENODEV;
2610
2611         cpufreq_global_kobject = kobject_create();
2612         BUG_ON(!cpufreq_global_kobject);
2613
2614         register_syscore_ops(&cpufreq_syscore_ops);
2615
2616         return 0;
2617 }
2618 core_initcall(cpufreq_core_init);