drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/syscore_ops.h>
  31 #include <linux/tick.h>
  32 #include <trace/events/power.h>
  33
  34 /**
  35  * The "cpufreq driver" - the arch- or hardware-dependent low
  36  * level driver of CPUFreq support, and its spinlock. This lock
  37  * also protects the cpufreq_cpu_data array.
  38  */
  39 static struct cpufreq_driver *cpufreq_driver;
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  42 static DEFINE_RWLOCK(cpufreq_driver_lock);
  43 DEFINE_MUTEX(cpufreq_governor_lock);
  44 static LIST_HEAD(cpufreq_policy_list);
  45
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48
  49 /* Flag to suspend/resume CPUFreq governors */
  50 static bool cpufreq_suspended;
  51
  52 static inline bool has_target(void)
  53 {
  54         return cpufreq_driver->target_index || cpufreq_driver->target;
  55 }
  56
  57 /*
  58  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  59  * sections
  60  */
  61 static DECLARE_RWSEM(cpufreq_rwsem);
  62
  63 /* internal prototypes */
  64 static int __cpufreq_governor(struct cpufreq_policy *policy,
  65                 unsigned int event);
  66 static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
  67 static void handle_update(struct work_struct *work);
  68
  69 /**
  70  * Two notifier lists: the "policy" list is involved in the
  71  * validation process for a new CPU frequency policy; the
  72  * "transition" list for kernel code that needs to handle
  73  * changes to devices when the CPU clock speed changes.
  74  * The mutex locks both lists.
  75  */
  76 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  77 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  78
  79 static bool init_cpufreq_transition_notifier_list_called;
  80 static int __init init_cpufreq_transition_notifier_list(void)
  81 {
  82         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  83         init_cpufreq_transition_notifier_list_called = true;
  84         return 0;
  85 }
  86 pure_initcall(init_cpufreq_transition_notifier_list);
  87
  88 static int off __read_mostly;
  89 static int cpufreq_disabled(void)
  90 {
  91         return off;
  92 }
  93 void disable_cpufreq(void)
  94 {
  95         off = 1;
  96 }
  97 static LIST_HEAD(cpufreq_governor_list);
  98 static DEFINE_MUTEX(cpufreq_governor_mutex);
  99
 100 bool have_governor_per_policy(void)
 101 {
 102         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 103 }
 104 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 105
 106 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 107 {
 108         if (have_governor_per_policy())
 109                 return &policy->kobj;
 110         else
 111                 return cpufreq_global_kobject;
 112 }
 113 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 114
 115 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 116 {
 117         u64 idle_time;
 118         u64 cur_wall_time;
 119         u64 busy_time;
 120
 121         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 122
 123         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 127         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 128         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 129
 130         idle_time = cur_wall_time - busy_time;
 131         if (wall)
 132                 *wall = cputime_to_usecs(cur_wall_time);
 133
 134         return cputime_to_usecs(idle_time);
 135 }
 136
 137 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 138 {
 139         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 140
 141         if (idle_time == -1ULL)
 142                 return get_cpu_idle_time_jiffy(cpu, wall);
 143         else if (!io_busy)
 144                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 145
 146         return idle_time;
 147 }
 148 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 149
 150 /*
 151  * This is a generic cpufreq init() routine which can be used by cpufreq
 152  * drivers of SMP systems. It will do following:
 153  * - validate & show freq table passed
 154  * - set policies transition latency
 155  * - policy->cpus with all possible CPUs
 156  */
 157 int cpufreq_generic_init(struct cpufreq_policy *policy,
 158                 struct cpufreq_frequency_table *table,
 159                 unsigned int transition_latency)
 160 {
 161         int ret;
 162
 163         ret = cpufreq_table_validate_and_show(policy, table);
 164         if (ret) {
 165                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 166                 return ret;
 167         }
 168
 169         policy->cpuinfo.transition_latency = transition_latency;
 170
 171         /*
 172          * The driver only supports the SMP configuartion where all processors
 173          * share the clock and voltage and clock.
 174          */
 175         cpumask_setall(policy->cpus);
 176
 177         return 0;
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 180
 181 unsigned int cpufreq_generic_get(unsigned int cpu)
 182 {
 183         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 184
 185         if (!policy || IS_ERR(policy->clk)) {
 186                 pr_err("%s: No %s associated to cpu: %d\n",
 187                        __func__, policy ? "clk" : "policy", cpu);
 188                 return 0;
 189         }
 190
 191         return clk_get_rate(policy->clk) / 1000;
 192 }
 193 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 194
 195 /* Only for cpufreq core internal use */
 196 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 197 {
 198         return per_cpu(cpufreq_cpu_data, cpu);
 199 }
 200
 201 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 202 {
 203         struct cpufreq_policy *policy = NULL;
 204         unsigned long flags;
 205
 206         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 207                 return NULL;
 208
 209         if (!down_read_trylock(&cpufreq_rwsem))
 210                 return NULL;
 211
 212         /* get the cpufreq driver */
 213         read_lock_irqsave(&cpufreq_driver_lock, flags);
 214
 215         if (cpufreq_driver) {
 216                 /* get the CPU */
 217                 policy = per_cpu(cpufreq_cpu_data, cpu);
 218                 if (policy)
 219                         kobject_get(&policy->kobj);
 220         }
 221
 222         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 223
 224         if (!policy)
 225                 up_read(&cpufreq_rwsem);
 226
 227         return policy;
 228 }
 229 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 230
 231 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 232 {
 233         if (cpufreq_disabled())
 234                 return;
 235
 236         kobject_put(&policy->kobj);
 237         up_read(&cpufreq_rwsem);
 238 }
 239 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 240
 241 /*********************************************************************
 242  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 243  *********************************************************************/
 244
 245 /**
 246  * adjust_jiffies - adjust the system "loops_per_jiffy"
 247  *
 248  * This function alters the system "loops_per_jiffy" for the clock
 249  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 250  * systems as each CPU might be scaled differently. So, use the arch
 251  * per-CPU loops_per_jiffy value wherever possible.
 252  */
 253 #ifndef CONFIG_SMP
 254 static unsigned long l_p_j_ref;
 255 static unsigned int l_p_j_ref_freq;
 256
 257 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 258 {
 259         if (ci->flags & CPUFREQ_CONST_LOOPS)
 260                 return;
 261
 262         if (!l_p_j_ref_freq) {
 263                 l_p_j_ref = loops_per_jiffy;
 264                 l_p_j_ref_freq = ci->old;
 265                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 266                          l_p_j_ref, l_p_j_ref_freq);
 267         }
 268         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 269                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 270                                                                 ci->new);
 271                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 272                          loops_per_jiffy, ci->new);
 273         }
 274 }
 275 #else
 276 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 277 {
 278         return;
 279 }
 280 #endif
 281
 282 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 283                 struct cpufreq_freqs *freqs, unsigned int state)
 284 {
 285         BUG_ON(irqs_disabled());
 286
 287         if (cpufreq_disabled())
 288                 return;
 289
 290         freqs->flags = cpufreq_driver->flags;
 291         pr_debug("notification %u of frequency transition to %u kHz\n",
 292                  state, freqs->new);
 293
 294         switch (state) {
 295
 296         case CPUFREQ_PRECHANGE:
 297                 /* detect if the driver reported a value as "old frequency"
 298                  * which is not equal to what the cpufreq core thinks is
 299                  * "old frequency".
 300                  */
 301                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 302                         if ((policy) && (policy->cpu == freqs->cpu) &&
 303                             (policy->cur) && (policy->cur != freqs->old)) {
 304                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 305                                          freqs->old, policy->cur);
 306                                 freqs->old = policy->cur;
 307                         }
 308                 }
 309                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 310                                 CPUFREQ_PRECHANGE, freqs);
 311                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 312                 break;
 313
 314         case CPUFREQ_POSTCHANGE:
 315                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 316                 pr_debug("FREQ: %lu - CPU: %lu\n",
 317                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 318                 trace_cpu_frequency(freqs->new, freqs->cpu);
 319                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 320                                 CPUFREQ_POSTCHANGE, freqs);
 321                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 322                         policy->cur = freqs->new;
 323                 break;
 324         }
 325 }
 326
 327 /**
 328  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 329  * on frequency transition.
 330  *
 331  * This function calls the transition notifiers and the "adjust_jiffies"
 332  * function. It is called twice on all CPU frequency changes that have
 333  * external effects.
 334  */
 335 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 336                 struct cpufreq_freqs *freqs, unsigned int state)
 337 {
 338         for_each_cpu(freqs->cpu, policy->cpus)
 339                 __cpufreq_notify_transition(policy, freqs, state);
 340 }
 341
 342 /* Do post notifications when there are chances that transition has failed */
 343 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 344                 struct cpufreq_freqs *freqs, int transition_failed)
 345 {
 346         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 347         if (!transition_failed)
 348                 return;
 349
 350         swap(freqs->old, freqs->new);
 351         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 352         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 353 }
 354
 355 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 356                 struct cpufreq_freqs *freqs)
 357 {
 358
 359         /*
 360          * Catch double invocations of _begin() which lead to self-deadlock.
 361          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 362          * doesn't invoke _begin() on their behalf, and hence the chances of
 363          * double invocations are very low. Moreover, there are scenarios
 364          * where these checks can emit false-positive warnings in these
 365          * drivers; so we avoid that by skipping them altogether.
 366          */
 367         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 368                                 && current == policy->transition_task);
 369
 370 wait:
 371         wait_event(policy->transition_wait, !policy->transition_ongoing);
 372
 373         spin_lock(&policy->transition_lock);
 374
 375         if (unlikely(policy->transition_ongoing)) {
 376                 spin_unlock(&policy->transition_lock);
 377                 goto wait;
 378         }
 379
 380         policy->transition_ongoing = true;
 381         policy->transition_task = current;
 382
 383         spin_unlock(&policy->transition_lock);
 384
 385         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 386 }
 387 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 388
 389 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 390                 struct cpufreq_freqs *freqs, int transition_failed)
 391 {
 392         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 393                 return;
 394
 395         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 396
 397         policy->transition_ongoing = false;
 398         policy->transition_task = NULL;
 399
 400         wake_up(&policy->transition_wait);
 401 }
 402 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 403
 404
 405 /*********************************************************************
 406  *                          SYSFS INTERFACE                          *
 407  *********************************************************************/
 408 static ssize_t show_boost(struct kobject *kobj,
 409                                  struct attribute *attr, char *buf)
 410 {
 411         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 412 }
 413
 414 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 415                                   const char *buf, size_t count)
 416 {
 417         int ret, enable;
 418
 419         ret = sscanf(buf, "%d", &enable);
 420         if (ret != 1 || enable < 0 || enable > 1)
 421                 return -EINVAL;
 422
 423         if (cpufreq_boost_trigger_state(enable)) {
 424                 pr_err("%s: Cannot %s BOOST!\n",
 425                        __func__, enable ? "enable" : "disable");
 426                 return -EINVAL;
 427         }
 428
 429         pr_debug("%s: cpufreq BOOST %s\n",
 430                  __func__, enable ? "enabled" : "disabled");
 431
 432         return count;
 433 }
 434 define_one_global_rw(boost);
 435
 436 static struct cpufreq_governor *find_governor(const char *str_governor)
 437 {
 438         struct cpufreq_governor *t;
 439
 440         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 441                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 442                         return t;
 443
 444         return NULL;
 445 }
 446
 447 /**
 448  * cpufreq_parse_governor - parse a governor string
 449  */
 450 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 451                                 struct cpufreq_governor **governor)
 452 {
 453         int err = -EINVAL;
 454
 455         if (!cpufreq_driver)
 456                 goto out;
 457
 458         if (cpufreq_driver->setpolicy) {
 459                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 460                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 461                         err = 0;
 462                 } else if (!strncasecmp(str_governor, "powersave",
 463                                                 CPUFREQ_NAME_LEN)) {
 464                         *policy = CPUFREQ_POLICY_POWERSAVE;
 465                         err = 0;
 466                 }
 467         } else {
 468                 struct cpufreq_governor *t;
 469
 470                 mutex_lock(&cpufreq_governor_mutex);
 471
 472                 t = find_governor(str_governor);
 473
 474                 if (t == NULL) {
 475                         int ret;
 476
 477                         mutex_unlock(&cpufreq_governor_mutex);
 478                         ret = request_module("cpufreq_%s", str_governor);
 479                         mutex_lock(&cpufreq_governor_mutex);
 480
 481                         if (ret == 0)
 482                                 t = find_governor(str_governor);
 483                 }
 484
 485                 if (t != NULL) {
 486                         *governor = t;
 487                         err = 0;
 488                 }
 489
 490                 mutex_unlock(&cpufreq_governor_mutex);
 491         }
 492 out:
 493         return err;
 494 }
 495
 496 /**
 497  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 498  * print out cpufreq information
 499  *
 500  * Write out information from cpufreq_driver->policy[cpu]; object must be
 501  * "unsigned int".
 502  */
 503
 504 #define show_one(file_name, object)                     \
 505 static ssize_t show_##file_name                         \
 506 (struct cpufreq_policy *policy, char *buf)              \
 507 {                                                       \
 508         return sprintf(buf, "%u\n", policy->object);    \
 509 }
 510
 511 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 512 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 513 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 514 show_one(scaling_min_freq, min);
 515 show_one(scaling_max_freq, max);
 516
 517 static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
 518 {
 519         ssize_t ret;
 520
 521         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 522                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 523         else
 524                 ret = sprintf(buf, "%u\n", policy->cur);
 525         return ret;
 526 }
 527
 528 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 529                                 struct cpufreq_policy *new_policy);
 530
 531 /**
 532  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 533  */
 534 #define store_one(file_name, object)                    \
 535 static ssize_t store_##file_name                                        \
 536 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 537 {                                                                       \
 538         int ret, temp;                                                  \
 539         struct cpufreq_policy new_policy;                               \
 540                                                                         \
 541         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 542         if (ret)                                                        \
 543                 return -EINVAL;                                         \
 544                                                                         \
 545         ret = sscanf(buf, "%u", &new_policy.object);                    \
 546         if (ret != 1)                                                   \
 547                 return -EINVAL;                                         \
 548                                                                         \
 549         temp = new_policy.object;                                       \
 550         ret = cpufreq_set_policy(policy, &new_policy);          \
 551         if (!ret)                                                       \
 552                 policy->user_policy.object = temp;                      \
 553                                                                         \
 554         return ret ? ret : count;                                       \
 555 }
 556
 557 store_one(scaling_min_freq, min);
 558 store_one(scaling_max_freq, max);
 559
 560 /**
 561  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 562  */
 563 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 564                                         char *buf)
 565 {
 566         unsigned int cur_freq = __cpufreq_get(policy);
 567         if (!cur_freq)
 568                 return sprintf(buf, "<unknown>");
 569         return sprintf(buf, "%u\n", cur_freq);
 570 }
 571
 572 /**
 573  * show_scaling_governor - show the current policy for the specified CPU
 574  */
 575 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 576 {
 577         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 578                 return sprintf(buf, "powersave\n");
 579         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 580                 return sprintf(buf, "performance\n");
 581         else if (policy->governor)
 582                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 583                                 policy->governor->name);
 584         return -EINVAL;
 585 }
 586
 587 /**
 588  * store_scaling_governor - store policy for the specified CPU
 589  */
 590 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 591                                         const char *buf, size_t count)
 592 {
 593         int ret;
 594         char    str_governor[16];
 595         struct cpufreq_policy new_policy;
 596
 597         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 598         if (ret)
 599                 return ret;
 600
 601         ret = sscanf(buf, "%15s", str_governor);
 602         if (ret != 1)
 603                 return -EINVAL;
 604
 605         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 606                                                 &new_policy.governor))
 607                 return -EINVAL;
 608
 609         ret = cpufreq_set_policy(policy, &new_policy);
 610
 611         policy->user_policy.policy = policy->policy;
 612         policy->user_policy.governor = policy->governor;
 613
 614         if (ret)
 615                 return ret;
 616         else
 617                 return count;
 618 }
 619
 620 /**
 621  * show_scaling_driver - show the cpufreq driver currently loaded
 622  */
 623 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 624 {
 625         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 626 }
 627
 628 /**
 629  * show_scaling_available_governors - show the available CPUfreq governors
 630  */
 631 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 632                                                 char *buf)
 633 {
 634         ssize_t i = 0;
 635         struct cpufreq_governor *t;
 636
 637         if (!has_target()) {
 638                 i += sprintf(buf, "performance powersave");
 639                 goto out;
 640         }
 641
 642         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 643                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 644                     - (CPUFREQ_NAME_LEN + 2)))
 645                         goto out;
 646                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 647         }
 648 out:
 649         i += sprintf(&buf[i], "\n");
 650         return i;
 651 }
 652
 653 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 654 {
 655         ssize_t i = 0;
 656         unsigned int cpu;
 657
 658         for_each_cpu(cpu, mask) {
 659                 if (i)
 660                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 661                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 662                 if (i >= (PAGE_SIZE - 5))
 663                         break;
 664         }
 665         i += sprintf(&buf[i], "\n");
 666         return i;
 667 }
 668 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 669
 670 /**
 671  * show_related_cpus - show the CPUs affected by each transition even if
 672  * hw coordination is in use
 673  */
 674 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 675 {
 676         return cpufreq_show_cpus(policy->related_cpus, buf);
 677 }
 678
 679 /**
 680  * show_affected_cpus - show the CPUs affected by each transition
 681  */
 682 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 683 {
 684         return cpufreq_show_cpus(policy->cpus, buf);
 685 }
 686
 687 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 688                                         const char *buf, size_t count)
 689 {
 690         unsigned int freq = 0;
 691         unsigned int ret;
 692
 693         if (!policy->governor || !policy->governor->store_setspeed)
 694                 return -EINVAL;
 695
 696         ret = sscanf(buf, "%u", &freq);
 697         if (ret != 1)
 698                 return -EINVAL;
 699
 700         policy->governor->store_setspeed(policy, freq);
 701
 702         return count;
 703 }
 704
 705 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 706 {
 707         if (!policy->governor || !policy->governor->show_setspeed)
 708                 return sprintf(buf, "<unsupported>\n");
 709
 710         return policy->governor->show_setspeed(policy, buf);
 711 }
 712
 713 /**
 714  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 715  */
 716 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 717 {
 718         unsigned int limit;
 719         int ret;
 720         if (cpufreq_driver->bios_limit) {
 721                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 722                 if (!ret)
 723                         return sprintf(buf, "%u\n", limit);
 724         }
 725         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 726 }
 727
 728 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 729 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 730 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 731 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 732 cpufreq_freq_attr_ro(scaling_available_governors);
 733 cpufreq_freq_attr_ro(scaling_driver);
 734 cpufreq_freq_attr_ro(scaling_cur_freq);
 735 cpufreq_freq_attr_ro(bios_limit);
 736 cpufreq_freq_attr_ro(related_cpus);
 737 cpufreq_freq_attr_ro(affected_cpus);
 738 cpufreq_freq_attr_rw(scaling_min_freq);
 739 cpufreq_freq_attr_rw(scaling_max_freq);
 740 cpufreq_freq_attr_rw(scaling_governor);
 741 cpufreq_freq_attr_rw(scaling_setspeed);
 742
 743 static struct attribute *default_attrs[] = {
 744         &cpuinfo_min_freq.attr,
 745         &cpuinfo_max_freq.attr,
 746         &cpuinfo_transition_latency.attr,
 747         &scaling_min_freq.attr,
 748         &scaling_max_freq.attr,
 749         &affected_cpus.attr,
 750         &related_cpus.attr,
 751         &scaling_governor.attr,
 752         &scaling_driver.attr,
 753         &scaling_available_governors.attr,
 754         &scaling_setspeed.attr,
 755         NULL
 756 };
 757
 758 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 759 #define to_attr(a) container_of(a, struct freq_attr, attr)
 760
 761 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 762 {
 763         struct cpufreq_policy *policy = to_policy(kobj);
 764         struct freq_attr *fattr = to_attr(attr);
 765         ssize_t ret;
 766
 767         if (!down_read_trylock(&cpufreq_rwsem))
 768                 return -EINVAL;
 769
 770         down_read(&policy->rwsem);
 771
 772         if (fattr->show)
 773                 ret = fattr->show(policy, buf);
 774         else
 775                 ret = -EIO;
 776
 777         up_read(&policy->rwsem);
 778         up_read(&cpufreq_rwsem);
 779
 780         return ret;
 781 }
 782
 783 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 784                      const char *buf, size_t count)
 785 {
 786         struct cpufreq_policy *policy = to_policy(kobj);
 787         struct freq_attr *fattr = to_attr(attr);
 788         ssize_t ret = -EINVAL;
 789
 790         get_online_cpus();
 791
 792         if (!cpu_online(policy->cpu))
 793                 goto unlock;
 794
 795         if (!down_read_trylock(&cpufreq_rwsem))
 796                 goto unlock;
 797
 798         down_write(&policy->rwsem);
 799
 800         if (fattr->store)
 801                 ret = fattr->store(policy, buf, count);
 802         else
 803                 ret = -EIO;
 804
 805         up_write(&policy->rwsem);
 806
 807         up_read(&cpufreq_rwsem);
 808 unlock:
 809         put_online_cpus();
 810
 811         return ret;
 812 }
 813
 814 static void cpufreq_sysfs_release(struct kobject *kobj)
 815 {
 816         struct cpufreq_policy *policy = to_policy(kobj);
 817         pr_debug("last reference is dropped\n");
 818         complete(&policy->kobj_unregister);
 819 }
 820
 821 static const struct sysfs_ops sysfs_ops = {
 822         .show   = show,
 823         .store  = store,
 824 };
 825
 826 static struct kobj_type ktype_cpufreq = {
 827         .sysfs_ops      = &sysfs_ops,
 828         .default_attrs  = default_attrs,
 829         .release        = cpufreq_sysfs_release,
 830 };
 831
 832 struct kobject *cpufreq_global_kobject;
 833 EXPORT_SYMBOL(cpufreq_global_kobject);
 834
 835 static int cpufreq_global_kobject_usage;
 836
 837 int cpufreq_get_global_kobject(void)
 838 {
 839         if (!cpufreq_global_kobject_usage++)
 840                 return kobject_add(cpufreq_global_kobject,
 841                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 842
 843         return 0;
 844 }
 845 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 846
 847 void cpufreq_put_global_kobject(void)
 848 {
 849         if (!--cpufreq_global_kobject_usage)
 850                 kobject_del(cpufreq_global_kobject);
 851 }
 852 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 853
 854 int cpufreq_sysfs_create_file(const struct attribute *attr)
 855 {
 856         int ret = cpufreq_get_global_kobject();
 857
 858         if (!ret) {
 859                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 860                 if (ret)
 861                         cpufreq_put_global_kobject();
 862         }
 863
 864         return ret;
 865 }
 866 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 867
 868 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 869 {
 870         sysfs_remove_file(cpufreq_global_kobject, attr);
 871         cpufreq_put_global_kobject();
 872 }
 873 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 874
 875 /* symlink affected CPUs */
 876 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 877 {
 878         unsigned int j;
 879         int ret = 0;
 880
 881         for_each_cpu(j, policy->cpus) {
 882                 struct device *cpu_dev;
 883
 884                 if (j == policy->cpu)
 885                         continue;
 886
 887                 pr_debug("Adding link for CPU: %u\n", j);
 888                 cpu_dev = get_cpu_device(j);
 889                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 890                                         "cpufreq");
 891                 if (ret)
 892                         break;
 893         }
 894         return ret;
 895 }
 896
 897 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 898                                      struct device *dev)
 899 {
 900         struct freq_attr **drv_attr;
 901         int ret = 0;
 902
 903         /* set up files for this cpu device */
 904         drv_attr = cpufreq_driver->attr;
 905         while (drv_attr && *drv_attr) {
 906                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 907                 if (ret)
 908                         return ret;
 909                 drv_attr++;
 910         }
 911         if (cpufreq_driver->get) {
 912                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 913                 if (ret)
 914                         return ret;
 915         }
 916
 917         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 918         if (ret)
 919                 return ret;
 920
 921         if (cpufreq_driver->bios_limit) {
 922                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 923                 if (ret)
 924                         return ret;
 925         }
 926
 927         return cpufreq_add_dev_symlink(policy);
 928 }
 929
 930 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 931 {
 932         struct cpufreq_governor *gov = NULL;
 933         struct cpufreq_policy new_policy;
 934         int ret = 0;
 935
 936         memcpy(&new_policy, policy, sizeof(*policy));
 937
 938         /* Update governor of new_policy to the governor used before hotplug */
 939         gov = find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
 940         if (gov)
 941                 pr_debug("Restoring governor %s for cpu %d\n",
 942                                 policy->governor->name, policy->cpu);
 943         else
 944                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 945
 946         new_policy.governor = gov;
 947
 948         /* Use the default policy if its valid. */
 949         if (cpufreq_driver->setpolicy)
 950                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 951
 952         /* set default policy */
 953         ret = cpufreq_set_policy(policy, &new_policy);
 954         if (ret) {
 955                 pr_debug("setting policy failed\n");
 956                 if (cpufreq_driver->exit)
 957                         cpufreq_driver->exit(policy);
 958         }
 959 }
 960
 961 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 962                                   unsigned int cpu, struct device *dev)
 963 {
 964         int ret = 0;
 965         unsigned long flags;
 966
 967         if (has_target()) {
 968                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 969                 if (ret) {
 970                         pr_err("%s: Failed to stop governor\n", __func__);
 971                         return ret;
 972                 }
 973         }
 974
 975         down_write(&policy->rwsem);
 976
 977         write_lock_irqsave(&cpufreq_driver_lock, flags);
 978
 979         cpumask_set_cpu(cpu, policy->cpus);
 980         per_cpu(cpufreq_cpu_data, cpu) = policy;
 981         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 982
 983         up_write(&policy->rwsem);
 984
 985         if (has_target()) {
 986                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
 987                 if (!ret)
 988                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 989
 990                 if (ret) {
 991                         pr_err("%s: Failed to start governor\n", __func__);
 992                         return ret;
 993                 }
 994         }
 995
 996         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 997 }
 998
 999 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1000 {
1001         struct cpufreq_policy *policy;
1002         unsigned long flags;
1003
1004         read_lock_irqsave(&cpufreq_driver_lock, flags);
1005
1006         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
1007
1008         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1009
1010         if (policy)
1011                 policy->governor = NULL;
1012
1013         return policy;
1014 }
1015
1016 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1017 {
1018         struct cpufreq_policy *policy;
1019
1020         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1021         if (!policy)
1022                 return NULL;
1023
1024         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1025                 goto err_free_policy;
1026
1027         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1028                 goto err_free_cpumask;
1029
1030         INIT_LIST_HEAD(&policy->policy_list);
1031         init_rwsem(&policy->rwsem);
1032         spin_lock_init(&policy->transition_lock);
1033         init_waitqueue_head(&policy->transition_wait);
1034
1035         return policy;
1036
1037 err_free_cpumask:
1038         free_cpumask_var(policy->cpus);
1039 err_free_policy:
1040         kfree(policy);
1041
1042         return NULL;
1043 }
1044
1045 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1046 {
1047         struct kobject *kobj;
1048         struct completion *cmp;
1049
1050         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1051                         CPUFREQ_REMOVE_POLICY, policy);
1052
1053         down_read(&policy->rwsem);
1054         kobj = &policy->kobj;
1055         cmp = &policy->kobj_unregister;
1056         up_read(&policy->rwsem);
1057         kobject_put(kobj);
1058
1059         /*
1060          * We need to make sure that the underlying kobj is
1061          * actually not referenced anymore by anybody before we
1062          * proceed with unloading.
1063          */
1064         pr_debug("waiting for dropping of refcount\n");
1065         wait_for_completion(cmp);
1066         pr_debug("wait complete\n");
1067 }
1068
1069 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1070 {
1071         free_cpumask_var(policy->related_cpus);
1072         free_cpumask_var(policy->cpus);
1073         kfree(policy);
1074 }
1075
1076 static int update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu,
1077                              struct device *cpu_dev)
1078 {
1079         int ret;
1080
1081         if (WARN_ON(cpu == policy->cpu))
1082                 return 0;
1083
1084         /* Move kobject to the new policy->cpu */
1085         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1086         if (ret) {
1087                 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1088                 return ret;
1089         }
1090
1091         down_write(&policy->rwsem);
1092
1093         policy->last_cpu = policy->cpu;
1094         policy->cpu = cpu;
1095
1096         up_write(&policy->rwsem);
1097
1098         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1099                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1100
1101         return 0;
1102 }
1103
1104 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1105 {
1106         unsigned int j, cpu = dev->id;
1107         int ret = -ENOMEM;
1108         struct cpufreq_policy *policy, *tpolicy;
1109         unsigned long flags;
1110         bool recover_policy = cpufreq_suspended;
1111
1112         if (cpu_is_offline(cpu))
1113                 return 0;
1114
1115         pr_debug("adding CPU %u\n", cpu);
1116
1117         /* check whether a different CPU already registered this
1118          * CPU because it is in the same boat. */
1119         policy = cpufreq_cpu_get(cpu);
1120         if (unlikely(policy)) {
1121                 cpufreq_cpu_put(policy);
1122                 return 0;
1123         }
1124
1125         if (!down_read_trylock(&cpufreq_rwsem))
1126                 return 0;
1127
1128         /* Check if this cpu was hot-unplugged earlier and has siblings */
1129         read_lock_irqsave(&cpufreq_driver_lock, flags);
1130         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1131                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1132                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1133                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1134                         up_read(&cpufreq_rwsem);
1135                         return ret;
1136                 }
1137         }
1138         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1139
1140         /*
1141          * Restore the saved policy when doing light-weight init and fall back
1142          * to the full init if that fails.
1143          */
1144         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1145         if (!policy) {
1146                 recover_policy = false;
1147                 policy = cpufreq_policy_alloc();
1148                 if (!policy)
1149                         goto nomem_out;
1150         }
1151
1152         /*
1153          * In the resume path, since we restore a saved policy, the assignment
1154          * to policy->cpu is like an update of the existing policy, rather than
1155          * the creation of a brand new one. So we need to perform this update
1156          * by invoking update_policy_cpu().
1157          */
1158         if (recover_policy && cpu != policy->cpu)
1159                 WARN_ON(update_policy_cpu(policy, cpu, dev));
1160         else
1161                 policy->cpu = cpu;
1162
1163         cpumask_copy(policy->cpus, cpumask_of(cpu));
1164
1165         init_completion(&policy->kobj_unregister);
1166         INIT_WORK(&policy->update, handle_update);
1167
1168         /* call driver. From then on the cpufreq must be able
1169          * to accept all calls to ->verify and ->setpolicy for this CPU
1170          */
1171         ret = cpufreq_driver->init(policy);
1172         if (ret) {
1173                 pr_debug("initialization failed\n");
1174                 goto err_set_policy_cpu;
1175         }
1176
1177         down_write(&policy->rwsem);
1178
1179         /* related cpus should atleast have policy->cpus */
1180         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1181
1182         /*
1183          * affected cpus must always be the one, which are online. We aren't
1184          * managing offline cpus here.
1185          */
1186         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1187
1188         if (!recover_policy) {
1189                 policy->user_policy.min = policy->min;
1190                 policy->user_policy.max = policy->max;
1191
1192                 /* prepare interface data */
1193                 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1194                                            &dev->kobj, "cpufreq");
1195                 if (ret) {
1196                         pr_err("%s: failed to init policy->kobj: %d\n",
1197                                __func__, ret);
1198                         goto err_init_policy_kobj;
1199                 }
1200         }
1201
1202         write_lock_irqsave(&cpufreq_driver_lock, flags);
1203         for_each_cpu(j, policy->cpus)
1204                 per_cpu(cpufreq_cpu_data, j) = policy;
1205         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1206
1207         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1208                 policy->cur = cpufreq_driver->get(policy->cpu);
1209                 if (!policy->cur) {
1210                         pr_err("%s: ->get() failed\n", __func__);
1211                         goto err_get_freq;
1212                 }
1213         }
1214
1215         /*
1216          * Sometimes boot loaders set CPU frequency to a value outside of
1217          * frequency table present with cpufreq core. In such cases CPU might be
1218          * unstable if it has to run on that frequency for long duration of time
1219          * and so its better to set it to a frequency which is specified in
1220          * freq-table. This also makes cpufreq stats inconsistent as
1221          * cpufreq-stats would fail to register because current frequency of CPU
1222          * isn't found in freq-table.
1223          *
1224          * Because we don't want this change to effect boot process badly, we go
1225          * for the next freq which is >= policy->cur ('cur' must be set by now,
1226          * otherwise we will end up setting freq to lowest of the table as 'cur'
1227          * is initialized to zero).
1228          *
1229          * We are passing target-freq as "policy->cur - 1" otherwise
1230          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1231          * equal to target-freq.
1232          */
1233         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1234             && has_target()) {
1235                 /* Are we running at unknown frequency ? */
1236                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1237                 if (ret == -EINVAL) {
1238                         /* Warn user and fix it */
1239                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1240                                 __func__, policy->cpu, policy->cur);
1241                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1242                                 CPUFREQ_RELATION_L);
1243
1244                         /*
1245                          * Reaching here after boot in a few seconds may not
1246                          * mean that system will remain stable at "unknown"
1247                          * frequency for longer duration. Hence, a BUG_ON().
1248                          */
1249                         BUG_ON(ret);
1250                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1251                                 __func__, policy->cpu, policy->cur);
1252                 }
1253         }
1254
1255         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1256                                      CPUFREQ_START, policy);
1257
1258         if (!recover_policy) {
1259                 ret = cpufreq_add_dev_interface(policy, dev);
1260                 if (ret)
1261                         goto err_out_unregister;
1262                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1263                                 CPUFREQ_CREATE_POLICY, policy);
1264         }
1265
1266         write_lock_irqsave(&cpufreq_driver_lock, flags);
1267         list_add(&policy->policy_list, &cpufreq_policy_list);
1268         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1269
1270         cpufreq_init_policy(policy);
1271
1272         if (!recover_policy) {
1273                 policy->user_policy.policy = policy->policy;
1274                 policy->user_policy.governor = policy->governor;
1275         }
1276         up_write(&policy->rwsem);
1277
1278         kobject_uevent(&policy->kobj, KOBJ_ADD);
1279
1280         up_read(&cpufreq_rwsem);
1281
1282         /* Callback for handling stuff after policy is ready */
1283         if (cpufreq_driver->ready)
1284                 cpufreq_driver->ready(policy);
1285
1286         pr_debug("initialization complete\n");
1287
1288         return 0;
1289
1290 err_out_unregister:
1291 err_get_freq:
1292         write_lock_irqsave(&cpufreq_driver_lock, flags);
1293         for_each_cpu(j, policy->cpus)
1294                 per_cpu(cpufreq_cpu_data, j) = NULL;
1295         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1296
1297         if (!recover_policy) {
1298                 kobject_put(&policy->kobj);
1299                 wait_for_completion(&policy->kobj_unregister);
1300         }
1301 err_init_policy_kobj:
1302         up_write(&policy->rwsem);
1303
1304         if (cpufreq_driver->exit)
1305                 cpufreq_driver->exit(policy);
1306 err_set_policy_cpu:
1307         if (recover_policy) {
1308                 /* Do not leave stale fallback data behind. */
1309                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1310                 cpufreq_policy_put_kobj(policy);
1311         }
1312         cpufreq_policy_free(policy);
1313
1314 nomem_out:
1315         up_read(&cpufreq_rwsem);
1316
1317         return ret;
1318 }
1319
1320 /**
1321  * cpufreq_add_dev - add a CPU device
1322  *
1323  * Adds the cpufreq interface for a CPU device.
1324  *
1325  * The Oracle says: try running cpufreq registration/unregistration concurrently
1326  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1327  * mess up, but more thorough testing is needed. - Mathieu
1328  */
1329 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1330 {
1331         return __cpufreq_add_dev(dev, sif);
1332 }
1333
1334 static int __cpufreq_remove_dev_prepare(struct device *dev,
1335                                         struct subsys_interface *sif)
1336 {
1337         unsigned int cpu = dev->id, cpus;
1338         int ret;
1339         unsigned long flags;
1340         struct cpufreq_policy *policy;
1341
1342         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1343
1344         write_lock_irqsave(&cpufreq_driver_lock, flags);
1345
1346         policy = per_cpu(cpufreq_cpu_data, cpu);
1347
1348         /* Save the policy somewhere when doing a light-weight tear-down */
1349         if (cpufreq_suspended)
1350                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1351
1352         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1353
1354         if (!policy) {
1355                 pr_debug("%s: No cpu_data found\n", __func__);
1356                 return -EINVAL;
1357         }
1358
1359         if (has_target()) {
1360                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1361                 if (ret) {
1362                         pr_err("%s: Failed to stop governor\n", __func__);
1363                         return ret;
1364                 }
1365
1366                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1367                         policy->governor->name, CPUFREQ_NAME_LEN);
1368         }
1369
1370         down_read(&policy->rwsem);
1371         cpus = cpumask_weight(policy->cpus);
1372         up_read(&policy->rwsem);
1373
1374         if (cpu != policy->cpu) {
1375                 sysfs_remove_link(&dev->kobj, "cpufreq");
1376         } else if (cpus > 1) {
1377                 /* Nominate new CPU */
1378                 int new_cpu = cpumask_any_but(policy->cpus, cpu);
1379                 struct device *cpu_dev = get_cpu_device(new_cpu);
1380
1381                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1382                 ret = update_policy_cpu(policy, new_cpu, cpu_dev);
1383                 if (ret) {
1384                         if (sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1385                                               "cpufreq"))
1386                                 pr_err("%s: Failed to restore kobj link to cpu:%d\n",
1387                                        __func__, cpu_dev->id);
1388                         return ret;
1389                 }
1390
1391                 if (!cpufreq_suspended)
1392                         pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1393                                  __func__, new_cpu, cpu);
1394         } else if (cpufreq_driver->stop_cpu) {
1395                 cpufreq_driver->stop_cpu(policy);
1396         }
1397
1398         return 0;
1399 }
1400
1401 static int __cpufreq_remove_dev_finish(struct device *dev,
1402                                        struct subsys_interface *sif)
1403 {
1404         unsigned int cpu = dev->id, cpus;
1405         int ret;
1406         unsigned long flags;
1407         struct cpufreq_policy *policy;
1408
1409         read_lock_irqsave(&cpufreq_driver_lock, flags);
1410         policy = per_cpu(cpufreq_cpu_data, cpu);
1411         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1412
1413         if (!policy) {
1414                 pr_debug("%s: No cpu_data found\n", __func__);
1415                 return -EINVAL;
1416         }
1417
1418         down_write(&policy->rwsem);
1419         cpus = cpumask_weight(policy->cpus);
1420
1421         if (cpus > 1)
1422                 cpumask_clear_cpu(cpu, policy->cpus);
1423         up_write(&policy->rwsem);
1424
1425         /* If cpu is last user of policy, free policy */
1426         if (cpus == 1) {
1427                 if (has_target()) {
1428                         ret = __cpufreq_governor(policy,
1429                                         CPUFREQ_GOV_POLICY_EXIT);
1430                         if (ret) {
1431                                 pr_err("%s: Failed to exit governor\n",
1432                                        __func__);
1433                                 return ret;
1434                         }
1435                 }
1436
1437                 if (!cpufreq_suspended)
1438                         cpufreq_policy_put_kobj(policy);
1439
1440                 /*
1441                  * Perform the ->exit() even during light-weight tear-down,
1442                  * since this is a core component, and is essential for the
1443                  * subsequent light-weight ->init() to succeed.
1444                  */
1445                 if (cpufreq_driver->exit)
1446                         cpufreq_driver->exit(policy);
1447
1448                 /* Remove policy from list of active policies */
1449                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1450                 list_del(&policy->policy_list);
1451                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1452
1453                 if (!cpufreq_suspended)
1454                         cpufreq_policy_free(policy);
1455         } else if (has_target()) {
1456                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1457                 if (!ret)
1458                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1459
1460                 if (ret) {
1461                         pr_err("%s: Failed to start governor\n", __func__);
1462                         return ret;
1463                 }
1464         }
1465
1466         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1467         return 0;
1468 }
1469
1470 /**
1471  * cpufreq_remove_dev - remove a CPU device
1472  *
1473  * Removes the cpufreq interface for a CPU device.
1474  */
1475 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1476 {
1477         unsigned int cpu = dev->id;
1478         int ret;
1479
1480         if (cpu_is_offline(cpu))
1481                 return 0;
1482
1483         ret = __cpufreq_remove_dev_prepare(dev, sif);
1484
1485         if (!ret)
1486                 ret = __cpufreq_remove_dev_finish(dev, sif);
1487
1488         return ret;
1489 }
1490
1491 static void handle_update(struct work_struct *work)
1492 {
1493         struct cpufreq_policy *policy =
1494                 container_of(work, struct cpufreq_policy, update);
1495         unsigned int cpu = policy->cpu;
1496         pr_debug("handle_update for cpu %u called\n", cpu);
1497         cpufreq_update_policy(cpu);
1498 }
1499
1500 /**
1501  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1502  *      in deep trouble.
1503  *      @policy: policy managing CPUs
1504  *      @new_freq: CPU frequency the CPU actually runs at
1505  *
1506  *      We adjust to current frequency first, and need to clean up later.
1507  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1508  */
1509 static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
1510                                 unsigned int new_freq)
1511 {
1512         struct cpufreq_freqs freqs;
1513
1514         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1515                  policy->cur, new_freq);
1516
1517         freqs.old = policy->cur;
1518         freqs.new = new_freq;
1519
1520         cpufreq_freq_transition_begin(policy, &freqs);
1521         cpufreq_freq_transition_end(policy, &freqs, 0);
1522 }
1523
1524 /**
1525  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1526  * @cpu: CPU number
1527  *
1528  * This is the last known freq, without actually getting it from the driver.
1529  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1530  */
1531 unsigned int cpufreq_quick_get(unsigned int cpu)
1532 {
1533         struct cpufreq_policy *policy;
1534         unsigned int ret_freq = 0;
1535
1536         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1537                 return cpufreq_driver->get(cpu);
1538
1539         policy = cpufreq_cpu_get(cpu);
1540         if (policy) {
1541                 ret_freq = policy->cur;
1542                 cpufreq_cpu_put(policy);
1543         }
1544
1545         return ret_freq;
1546 }
1547 EXPORT_SYMBOL(cpufreq_quick_get);
1548
1549 /**
1550  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1551  * @cpu: CPU number
1552  *
1553  * Just return the max possible frequency for a given CPU.
1554  */
1555 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1556 {
1557         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1558         unsigned int ret_freq = 0;
1559
1560         if (policy) {
1561                 ret_freq = policy->max;
1562                 cpufreq_cpu_put(policy);
1563         }
1564
1565         return ret_freq;
1566 }
1567 EXPORT_SYMBOL(cpufreq_quick_get_max);
1568
1569 static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1570 {
1571         unsigned int ret_freq = 0;
1572
1573         if (!cpufreq_driver->get)
1574                 return ret_freq;
1575
1576         ret_freq = cpufreq_driver->get(policy->cpu);
1577
1578         if (ret_freq && policy->cur &&
1579                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1580                 /* verify no discrepancy between actual and
1581                                         saved value exists */
1582                 if (unlikely(ret_freq != policy->cur)) {
1583                         cpufreq_out_of_sync(policy, ret_freq);
1584                         schedule_work(&policy->update);
1585                 }
1586         }
1587
1588         return ret_freq;
1589 }
1590
1591 /**
1592  * cpufreq_get - get the current CPU frequency (in kHz)
1593  * @cpu: CPU number
1594  *
1595  * Get the CPU current (static) CPU frequency
1596  */
1597 unsigned int cpufreq_get(unsigned int cpu)
1598 {
1599         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1600         unsigned int ret_freq = 0;
1601
1602         if (policy) {
1603                 down_read(&policy->rwsem);
1604                 ret_freq = __cpufreq_get(policy);
1605                 up_read(&policy->rwsem);
1606
1607                 cpufreq_cpu_put(policy);
1608         }
1609
1610         return ret_freq;
1611 }
1612 EXPORT_SYMBOL(cpufreq_get);
1613
1614 static struct subsys_interface cpufreq_interface = {
1615         .name           = "cpufreq",
1616         .subsys         = &cpu_subsys,
1617         .add_dev        = cpufreq_add_dev,
1618         .remove_dev     = cpufreq_remove_dev,
1619 };
1620
1621 /*
1622  * In case platform wants some specific frequency to be configured
1623  * during suspend..
1624  */
1625 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1626 {
1627         int ret;
1628
1629         if (!policy->suspend_freq) {
1630                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1631                 return -EINVAL;
1632         }
1633
1634         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1635                         policy->suspend_freq);
1636
1637         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1638                         CPUFREQ_RELATION_H);
1639         if (ret)
1640                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1641                                 __func__, policy->suspend_freq, ret);
1642
1643         return ret;
1644 }
1645 EXPORT_SYMBOL(cpufreq_generic_suspend);
1646
1647 /**
1648  * cpufreq_suspend() - Suspend CPUFreq governors
1649  *
1650  * Called during system wide Suspend/Hibernate cycles for suspending governors
1651  * as some platforms can't change frequency after this point in suspend cycle.
1652  * Because some of the devices (like: i2c, regulators, etc) they use for
1653  * changing frequency are suspended quickly after this point.
1654  */
1655 void cpufreq_suspend(void)
1656 {
1657         struct cpufreq_policy *policy;
1658
1659         if (!cpufreq_driver)
1660                 return;
1661
1662         if (!has_target())
1663                 goto suspend;
1664
1665         pr_debug("%s: Suspending Governors\n", __func__);
1666
1667         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1668                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1669                         pr_err("%s: Failed to stop governor for policy: %p\n",
1670                                 __func__, policy);
1671                 else if (cpufreq_driver->suspend
1672                     && cpufreq_driver->suspend(policy))
1673                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1674                                 policy);
1675         }
1676
1677 suspend:
1678         cpufreq_suspended = true;
1679 }
1680
1681 /**
1682  * cpufreq_resume() - Resume CPUFreq governors
1683  *
1684  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1685  * are suspended with cpufreq_suspend().
1686  */
1687 void cpufreq_resume(void)
1688 {
1689         struct cpufreq_policy *policy;
1690
1691         if (!cpufreq_driver)
1692                 return;
1693
1694         cpufreq_suspended = false;
1695
1696         if (!has_target())
1697                 return;
1698
1699         pr_debug("%s: Resuming Governors\n", __func__);
1700
1701         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1702                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1703                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1704                                 policy);
1705                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1706                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1707                         pr_err("%s: Failed to start governor for policy: %p\n",
1708                                 __func__, policy);
1709
1710                 /*
1711                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1712                  * policy in list. It will verify that the current freq is in
1713                  * sync with what we believe it to be.
1714                  */
1715                 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1716                         schedule_work(&policy->update);
1717         }
1718 }
1719
1720 /**
1721  *      cpufreq_get_current_driver - return current driver's name
1722  *
1723  *      Return the name string of the currently loaded cpufreq driver
1724  *      or NULL, if none.
1725  */
1726 const char *cpufreq_get_current_driver(void)
1727 {
1728         if (cpufreq_driver)
1729                 return cpufreq_driver->name;
1730
1731         return NULL;
1732 }
1733 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1734
1735 /**
1736  *      cpufreq_get_driver_data - return current driver data
1737  *
1738  *      Return the private data of the currently loaded cpufreq
1739  *      driver, or NULL if no cpufreq driver is loaded.
1740  */
1741 void *cpufreq_get_driver_data(void)
1742 {
1743         if (cpufreq_driver)
1744                 return cpufreq_driver->driver_data;
1745
1746         return NULL;
1747 }
1748 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1749
1750 /*********************************************************************
1751  *                     NOTIFIER LISTS INTERFACE                      *
1752  *********************************************************************/
1753
1754 /**
1755  *      cpufreq_register_notifier - register a driver with cpufreq
1756  *      @nb: notifier function to register
1757  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1758  *
1759  *      Add a driver to one of two lists: either a list of drivers that
1760  *      are notified about clock rate changes (once before and once after
1761  *      the transition), or a list of drivers that are notified about
1762  *      changes in cpufreq policy.
1763  *
1764  *      This function may sleep, and has the same return conditions as
1765  *      blocking_notifier_chain_register.
1766  */
1767 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1768 {
1769         int ret;
1770
1771         if (cpufreq_disabled())
1772                 return -EINVAL;
1773
1774         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1775
1776         switch (list) {
1777         case CPUFREQ_TRANSITION_NOTIFIER:
1778                 ret = srcu_notifier_chain_register(
1779                                 &cpufreq_transition_notifier_list, nb);
1780                 break;
1781         case CPUFREQ_POLICY_NOTIFIER:
1782                 ret = blocking_notifier_chain_register(
1783                                 &cpufreq_policy_notifier_list, nb);
1784                 break;
1785         default:
1786                 ret = -EINVAL;
1787         }
1788
1789         return ret;
1790 }
1791 EXPORT_SYMBOL(cpufreq_register_notifier);
1792
1793 /**
1794  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1795  *      @nb: notifier block to be unregistered
1796  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1797  *
1798  *      Remove a driver from the CPU frequency notifier list.
1799  *
1800  *      This function may sleep, and has the same return conditions as
1801  *      blocking_notifier_chain_unregister.
1802  */
1803 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1804 {
1805         int ret;
1806
1807         if (cpufreq_disabled())
1808                 return -EINVAL;
1809
1810         switch (list) {
1811         case CPUFREQ_TRANSITION_NOTIFIER:
1812                 ret = srcu_notifier_chain_unregister(
1813                                 &cpufreq_transition_notifier_list, nb);
1814                 break;
1815         case CPUFREQ_POLICY_NOTIFIER:
1816                 ret = blocking_notifier_chain_unregister(
1817                                 &cpufreq_policy_notifier_list, nb);
1818                 break;
1819         default:
1820                 ret = -EINVAL;
1821         }
1822
1823         return ret;
1824 }
1825 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1826
1827
1828 /*********************************************************************
1829  *                              GOVERNORS                            *
1830  *********************************************************************/
1831
1832 /* Must set freqs->new to intermediate frequency */
1833 static int __target_intermediate(struct cpufreq_policy *policy,
1834                                  struct cpufreq_freqs *freqs, int index)
1835 {
1836         int ret;
1837
1838         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1839
1840         /* We don't need to switch to intermediate freq */
1841         if (!freqs->new)
1842                 return 0;
1843
1844         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1845                  __func__, policy->cpu, freqs->old, freqs->new);
1846
1847         cpufreq_freq_transition_begin(policy, freqs);
1848         ret = cpufreq_driver->target_intermediate(policy, index);
1849         cpufreq_freq_transition_end(policy, freqs, ret);
1850
1851         if (ret)
1852                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1853                        __func__, ret);
1854
1855         return ret;
1856 }
1857
1858 static int __target_index(struct cpufreq_policy *policy,
1859                           struct cpufreq_frequency_table *freq_table, int index)
1860 {
1861         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1862         unsigned int intermediate_freq = 0;
1863         int retval = -EINVAL;
1864         bool notify;
1865
1866         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1867         if (notify) {
1868                 /* Handle switching to intermediate frequency */
1869                 if (cpufreq_driver->get_intermediate) {
1870                         retval = __target_intermediate(policy, &freqs, index);
1871                         if (retval)
1872                                 return retval;
1873
1874                         intermediate_freq = freqs.new;
1875                         /* Set old freq to intermediate */
1876                         if (intermediate_freq)
1877                                 freqs.old = freqs.new;
1878                 }
1879
1880                 freqs.new = freq_table[index].frequency;
1881                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1882                          __func__, policy->cpu, freqs.old, freqs.new);
1883
1884                 cpufreq_freq_transition_begin(policy, &freqs);
1885         }
1886
1887         retval = cpufreq_driver->target_index(policy, index);
1888         if (retval)
1889                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1890                        retval);
1891
1892         if (notify) {
1893                 cpufreq_freq_transition_end(policy, &freqs, retval);
1894
1895                 /*
1896                  * Failed after setting to intermediate freq? Driver should have
1897                  * reverted back to initial frequency and so should we. Check
1898                  * here for intermediate_freq instead of get_intermediate, in
1899                  * case we have't switched to intermediate freq at all.
1900                  */
1901                 if (unlikely(retval && intermediate_freq)) {
1902                         freqs.old = intermediate_freq;
1903                         freqs.new = policy->restore_freq;
1904                         cpufreq_freq_transition_begin(policy, &freqs);
1905                         cpufreq_freq_transition_end(policy, &freqs, 0);
1906                 }
1907         }
1908
1909         return retval;
1910 }
1911
1912 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1913                             unsigned int target_freq,
1914                             unsigned int relation)
1915 {
1916         unsigned int old_target_freq = target_freq;
1917         int retval = -EINVAL;
1918
1919         if (cpufreq_disabled())
1920                 return -ENODEV;
1921
1922         /* Make sure that target_freq is within supported range */
1923         if (target_freq > policy->max)
1924                 target_freq = policy->max;
1925         if (target_freq < policy->min)
1926                 target_freq = policy->min;
1927
1928         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1929                  policy->cpu, target_freq, relation, old_target_freq);
1930
1931         /*
1932          * This might look like a redundant call as we are checking it again
1933          * after finding index. But it is left intentionally for cases where
1934          * exactly same freq is called again and so we can save on few function
1935          * calls.
1936          */
1937         if (target_freq == policy->cur)
1938                 return 0;
1939
1940         /* Save last value to restore later on errors */
1941         policy->restore_freq = policy->cur;
1942
1943         if (cpufreq_driver->target)
1944                 retval = cpufreq_driver->target(policy, target_freq, relation);
1945         else if (cpufreq_driver->target_index) {
1946                 struct cpufreq_frequency_table *freq_table;
1947                 int index;
1948
1949                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1950                 if (unlikely(!freq_table)) {
1951                         pr_err("%s: Unable to find freq_table\n", __func__);
1952                         goto out;
1953                 }
1954
1955                 retval = cpufreq_frequency_table_target(policy, freq_table,
1956                                 target_freq, relation, &index);
1957                 if (unlikely(retval)) {
1958                         pr_err("%s: Unable to find matching freq\n", __func__);
1959                         goto out;
1960                 }
1961
1962                 if (freq_table[index].frequency == policy->cur) {
1963                         retval = 0;
1964                         goto out;
1965                 }
1966
1967                 retval = __target_index(policy, freq_table, index);
1968         }
1969
1970 out:
1971         return retval;
1972 }
1973 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1974
1975 int cpufreq_driver_target(struct cpufreq_policy *policy,
1976                           unsigned int target_freq,
1977                           unsigned int relation)
1978 {
1979         int ret = -EINVAL;
1980
1981         down_write(&policy->rwsem);
1982
1983         ret = __cpufreq_driver_target(policy, target_freq, relation);
1984
1985         up_write(&policy->rwsem);
1986
1987         return ret;
1988 }
1989 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1990
1991 static int __cpufreq_governor(struct cpufreq_policy *policy,
1992                                         unsigned int event)
1993 {
1994         int ret;
1995
1996         /* Only must be defined when default governor is known to have latency
1997            restrictions, like e.g. conservative or ondemand.
1998            That this is the case is already ensured in Kconfig
1999         */
2000 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2001         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2002 #else
2003         struct cpufreq_governor *gov = NULL;
2004 #endif
2005
2006         /* Don't start any governor operations if we are entering suspend */
2007         if (cpufreq_suspended)
2008                 return 0;
2009         /*
2010          * Governor might not be initiated here if ACPI _PPC changed
2011          * notification happened, so check it.
2012          */
2013         if (!policy->governor)
2014                 return -EINVAL;
2015
2016         if (policy->governor->max_transition_latency &&
2017             policy->cpuinfo.transition_latency >
2018             policy->governor->max_transition_latency) {
2019                 if (!gov)
2020                         return -EINVAL;
2021                 else {
2022                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2023                                 policy->governor->name, gov->name);
2024                         policy->governor = gov;
2025                 }
2026         }
2027
2028         if (event == CPUFREQ_GOV_POLICY_INIT)
2029                 if (!try_module_get(policy->governor->owner))
2030                         return -EINVAL;
2031
2032         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
2033                  policy->cpu, event);
2034
2035         mutex_lock(&cpufreq_governor_lock);
2036         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2037             || (!policy->governor_enabled
2038             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2039                 mutex_unlock(&cpufreq_governor_lock);
2040                 return -EBUSY;
2041         }
2042
2043         if (event == CPUFREQ_GOV_STOP)
2044                 policy->governor_enabled = false;
2045         else if (event == CPUFREQ_GOV_START)
2046                 policy->governor_enabled = true;
2047
2048         mutex_unlock(&cpufreq_governor_lock);
2049
2050         ret = policy->governor->governor(policy, event);
2051
2052         if (!ret) {
2053                 if (event == CPUFREQ_GOV_POLICY_INIT)
2054                         policy->governor->initialized++;
2055                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2056                         policy->governor->initialized--;
2057         } else {
2058                 /* Restore original values */
2059                 mutex_lock(&cpufreq_governor_lock);
2060                 if (event == CPUFREQ_GOV_STOP)
2061                         policy->governor_enabled = true;
2062                 else if (event == CPUFREQ_GOV_START)
2063                         policy->governor_enabled = false;
2064                 mutex_unlock(&cpufreq_governor_lock);
2065         }
2066
2067         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2068                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2069                 module_put(policy->governor->owner);
2070
2071         return ret;
2072 }
2073
2074 int cpufreq_register_governor(struct cpufreq_governor *governor)
2075 {
2076         int err;
2077
2078         if (!governor)
2079                 return -EINVAL;
2080
2081         if (cpufreq_disabled())
2082                 return -ENODEV;
2083
2084         mutex_lock(&cpufreq_governor_mutex);
2085
2086         governor->initialized = 0;
2087         err = -EBUSY;
2088         if (!find_governor(governor->name)) {
2089                 err = 0;
2090                 list_add(&governor->governor_list, &cpufreq_governor_list);
2091         }
2092
2093         mutex_unlock(&cpufreq_governor_mutex);
2094         return err;
2095 }
2096 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2097
2098 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2099 {
2100         int cpu;
2101
2102         if (!governor)
2103                 return;
2104
2105         if (cpufreq_disabled())
2106                 return;
2107
2108         for_each_present_cpu(cpu) {
2109                 if (cpu_online(cpu))
2110                         continue;
2111                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2112                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2113         }
2114
2115         mutex_lock(&cpufreq_governor_mutex);
2116         list_del(&governor->governor_list);
2117         mutex_unlock(&cpufreq_governor_mutex);
2118         return;
2119 }
2120 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2121
2122
2123 /*********************************************************************
2124  *                          POLICY INTERFACE                         *
2125  *********************************************************************/
2126
2127 /**
2128  * cpufreq_get_policy - get the current cpufreq_policy
2129  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2130  *      is written
2131  *
2132  * Reads the current cpufreq policy.
2133  */
2134 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2135 {
2136         struct cpufreq_policy *cpu_policy;
2137         if (!policy)
2138                 return -EINVAL;
2139
2140         cpu_policy = cpufreq_cpu_get(cpu);
2141         if (!cpu_policy)
2142                 return -EINVAL;
2143
2144         memcpy(policy, cpu_policy, sizeof(*policy));
2145
2146         cpufreq_cpu_put(cpu_policy);
2147         return 0;
2148 }
2149 EXPORT_SYMBOL(cpufreq_get_policy);
2150
2151 /*
2152  * policy : current policy.
2153  * new_policy: policy to be set.
2154  */
2155 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2156                                 struct cpufreq_policy *new_policy)
2157 {
2158         struct cpufreq_governor *old_gov;
2159         int ret;
2160
2161         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2162                  new_policy->cpu, new_policy->min, new_policy->max);
2163
2164         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2165
2166         if (new_policy->min > policy->max || new_policy->max < policy->min)
2167                 return -EINVAL;
2168
2169         /* verify the cpu speed can be set within this limit */
2170         ret = cpufreq_driver->verify(new_policy);
2171         if (ret)
2172                 return ret;
2173
2174         /* adjust if necessary - all reasons */
2175         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2176                         CPUFREQ_ADJUST, new_policy);
2177
2178         /* adjust if necessary - hardware incompatibility*/
2179         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2180                         CPUFREQ_INCOMPATIBLE, new_policy);
2181
2182         /*
2183          * verify the cpu speed can be set within this limit, which might be
2184          * different to the first one
2185          */
2186         ret = cpufreq_driver->verify(new_policy);
2187         if (ret)
2188                 return ret;
2189
2190         /* notification of the new policy */
2191         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2192                         CPUFREQ_NOTIFY, new_policy);
2193
2194         policy->min = new_policy->min;
2195         policy->max = new_policy->max;
2196
2197         pr_debug("new min and max freqs are %u - %u kHz\n",
2198                  policy->min, policy->max);
2199
2200         if (cpufreq_driver->setpolicy) {
2201                 policy->policy = new_policy->policy;
2202                 pr_debug("setting range\n");
2203                 return cpufreq_driver->setpolicy(new_policy);
2204         }
2205
2206         if (new_policy->governor == policy->governor)
2207                 goto out;
2208
2209         pr_debug("governor switch\n");
2210
2211         /* save old, working values */
2212         old_gov = policy->governor;
2213         /* end old governor */
2214         if (old_gov) {
2215                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2216                 up_write(&policy->rwsem);
2217                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2218                 down_write(&policy->rwsem);
2219         }
2220
2221         /* start new governor */
2222         policy->governor = new_policy->governor;
2223         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2224                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2225                         goto out;
2226
2227                 up_write(&policy->rwsem);
2228                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2229                 down_write(&policy->rwsem);
2230         }
2231
2232         /* new governor failed, so re-start old one */
2233         pr_debug("starting governor %s failed\n", policy->governor->name);
2234         if (old_gov) {
2235                 policy->governor = old_gov;
2236                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2237                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2238         }
2239
2240         return -EINVAL;
2241
2242  out:
2243         pr_debug("governor: change or update limits\n");
2244         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2245 }
2246
2247 /**
2248  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2249  *      @cpu: CPU which shall be re-evaluated
2250  *
2251  *      Useful for policy notifiers which have different necessities
2252  *      at different times.
2253  */
2254 int cpufreq_update_policy(unsigned int cpu)
2255 {
2256         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2257         struct cpufreq_policy new_policy;
2258         int ret;
2259
2260         if (!policy)
2261                 return -ENODEV;
2262
2263         down_write(&policy->rwsem);
2264
2265         pr_debug("updating policy for CPU %u\n", cpu);
2266         memcpy(&new_policy, policy, sizeof(*policy));
2267         new_policy.min = policy->user_policy.min;
2268         new_policy.max = policy->user_policy.max;
2269         new_policy.policy = policy->user_policy.policy;
2270         new_policy.governor = policy->user_policy.governor;
2271
2272         /*
2273          * BIOS might change freq behind our back
2274          * -> ask driver for current freq and notify governors about a change
2275          */
2276         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2277                 new_policy.cur = cpufreq_driver->get(cpu);
2278                 if (WARN_ON(!new_policy.cur)) {
2279                         ret = -EIO;
2280                         goto unlock;
2281                 }
2282
2283                 if (!policy->cur) {
2284                         pr_debug("Driver did not initialize current freq\n");
2285                         policy->cur = new_policy.cur;
2286                 } else {
2287                         if (policy->cur != new_policy.cur && has_target())
2288                                 cpufreq_out_of_sync(policy, new_policy.cur);
2289                 }
2290         }
2291
2292         ret = cpufreq_set_policy(policy, &new_policy);
2293
2294 unlock:
2295         up_write(&policy->rwsem);
2296
2297         cpufreq_cpu_put(policy);
2298         return ret;
2299 }
2300 EXPORT_SYMBOL(cpufreq_update_policy);
2301
2302 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2303                                         unsigned long action, void *hcpu)
2304 {
2305         unsigned int cpu = (unsigned long)hcpu;
2306         struct device *dev;
2307
2308         dev = get_cpu_device(cpu);
2309         if (dev) {
2310                 switch (action & ~CPU_TASKS_FROZEN) {
2311                 case CPU_ONLINE:
2312                         __cpufreq_add_dev(dev, NULL);
2313                         break;
2314
2315                 case CPU_DOWN_PREPARE:
2316                         __cpufreq_remove_dev_prepare(dev, NULL);
2317                         break;
2318
2319                 case CPU_POST_DEAD:
2320                         __cpufreq_remove_dev_finish(dev, NULL);
2321                         break;
2322
2323                 case CPU_DOWN_FAILED:
2324                         __cpufreq_add_dev(dev, NULL);
2325                         break;
2326                 }
2327         }
2328         return NOTIFY_OK;
2329 }
2330
2331 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2332         .notifier_call = cpufreq_cpu_callback,
2333 };
2334
2335 /*********************************************************************
2336  *               BOOST                                               *
2337  *********************************************************************/
2338 static int cpufreq_boost_set_sw(int state)
2339 {
2340         struct cpufreq_frequency_table *freq_table;
2341         struct cpufreq_policy *policy;
2342         int ret = -EINVAL;
2343
2344         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2345                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2346                 if (freq_table) {
2347                         ret = cpufreq_frequency_table_cpuinfo(policy,
2348                                                         freq_table);
2349                         if (ret) {
2350                                 pr_err("%s: Policy frequency update failed\n",
2351                                        __func__);
2352                                 break;
2353                         }
2354                         policy->user_policy.max = policy->max;
2355                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2356                 }
2357         }
2358
2359         return ret;
2360 }
2361
2362 int cpufreq_boost_trigger_state(int state)
2363 {
2364         unsigned long flags;
2365         int ret = 0;
2366
2367         if (cpufreq_driver->boost_enabled == state)
2368                 return 0;
2369
2370         write_lock_irqsave(&cpufreq_driver_lock, flags);
2371         cpufreq_driver->boost_enabled = state;
2372         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2373
2374         ret = cpufreq_driver->set_boost(state);
2375         if (ret) {
2376                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2377                 cpufreq_driver->boost_enabled = !state;
2378                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2379
2380                 pr_err("%s: Cannot %s BOOST\n",
2381                        __func__, state ? "enable" : "disable");
2382         }
2383
2384         return ret;
2385 }
2386
2387 int cpufreq_boost_supported(void)
2388 {
2389         if (likely(cpufreq_driver))
2390                 return cpufreq_driver->boost_supported;
2391
2392         return 0;
2393 }
2394 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2395
2396 int cpufreq_boost_enabled(void)
2397 {
2398         return cpufreq_driver->boost_enabled;
2399 }
2400 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2401
2402 /*********************************************************************
2403  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2404  *********************************************************************/
2405
2406 /**
2407  * cpufreq_register_driver - register a CPU Frequency driver
2408  * @driver_data: A struct cpufreq_driver containing the values#
2409  * submitted by the CPU Frequency driver.
2410  *
2411  * Registers a CPU Frequency driver to this core code. This code
2412  * returns zero on success, -EBUSY when another driver got here first
2413  * (and isn't unregistered in the meantime).
2414  *
2415  */
2416 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2417 {
2418         unsigned long flags;
2419         int ret;
2420
2421         if (cpufreq_disabled())
2422                 return -ENODEV;
2423
2424         if (!driver_data || !driver_data->verify || !driver_data->init ||
2425             !(driver_data->setpolicy || driver_data->target_index ||
2426                     driver_data->target) ||
2427              (driver_data->setpolicy && (driver_data->target_index ||
2428                     driver_data->target)) ||
2429              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2430                 return -EINVAL;
2431
2432         pr_debug("trying to register driver %s\n", driver_data->name);
2433
2434         write_lock_irqsave(&cpufreq_driver_lock, flags);
2435         if (cpufreq_driver) {
2436                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2437                 return -EEXIST;
2438         }
2439         cpufreq_driver = driver_data;
2440         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2441
2442         if (driver_data->setpolicy)
2443                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2444
2445         if (cpufreq_boost_supported()) {
2446                 /*
2447                  * Check if driver provides function to enable boost -
2448                  * if not, use cpufreq_boost_set_sw as default
2449                  */
2450                 if (!cpufreq_driver->set_boost)
2451                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2452
2453                 ret = cpufreq_sysfs_create_file(&boost.attr);
2454                 if (ret) {
2455                         pr_err("%s: cannot register global BOOST sysfs file\n",
2456                                __func__);
2457                         goto err_null_driver;
2458                 }
2459         }
2460
2461         ret = subsys_interface_register(&cpufreq_interface);
2462         if (ret)
2463                 goto err_boost_unreg;
2464
2465         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2466                 int i;
2467                 ret = -ENODEV;
2468
2469                 /* check for at least one working CPU */
2470                 for (i = 0; i < nr_cpu_ids; i++)
2471                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2472                                 ret = 0;
2473                                 break;
2474                         }
2475
2476                 /* if all ->init() calls failed, unregister */
2477                 if (ret) {
2478                         pr_debug("no CPU initialized for driver %s\n",
2479                                  driver_data->name);
2480                         goto err_if_unreg;
2481                 }
2482         }
2483
2484         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2485         pr_debug("driver %s up and running\n", driver_data->name);
2486
2487         return 0;
2488 err_if_unreg:
2489         subsys_interface_unregister(&cpufreq_interface);
2490 err_boost_unreg:
2491         if (cpufreq_boost_supported())
2492                 cpufreq_sysfs_remove_file(&boost.attr);
2493 err_null_driver:
2494         write_lock_irqsave(&cpufreq_driver_lock, flags);
2495         cpufreq_driver = NULL;
2496         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2497         return ret;
2498 }
2499 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2500
2501 /**
2502  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2503  *
2504  * Unregister the current CPUFreq driver. Only call this if you have
2505  * the right to do so, i.e. if you have succeeded in initialising before!
2506  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2507  * currently not initialised.
2508  */
2509 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2510 {
2511         unsigned long flags;
2512
2513         if (!cpufreq_driver || (driver != cpufreq_driver))
2514                 return -EINVAL;
2515
2516         pr_debug("unregistering driver %s\n", driver->name);
2517
2518         subsys_interface_unregister(&cpufreq_interface);
2519         if (cpufreq_boost_supported())
2520                 cpufreq_sysfs_remove_file(&boost.attr);
2521
2522         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2523
2524         down_write(&cpufreq_rwsem);
2525         write_lock_irqsave(&cpufreq_driver_lock, flags);
2526
2527         cpufreq_driver = NULL;
2528
2529         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2530         up_write(&cpufreq_rwsem);
2531
2532         return 0;
2533 }
2534 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2535
2536 /*
2537  * Stop cpufreq at shutdown to make sure it isn't holding any locks
2538  * or mutexes when secondary CPUs are halted.
2539  */
2540 static struct syscore_ops cpufreq_syscore_ops = {
2541         .shutdown = cpufreq_suspend,
2542 };
2543
2544 static int __init cpufreq_core_init(void)
2545 {
2546         if (cpufreq_disabled())
2547                 return -ENODEV;
2548
2549         cpufreq_global_kobject = kobject_create();
2550         BUG_ON(!cpufreq_global_kobject);
2551
2552         register_syscore_ops(&cpufreq_syscore_ops);
2553
2554         return 0;
2555 }
2556 core_initcall(cpufreq_core_init);