drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <asm/cputime.h>
  21 #include <linux/kernel.h>
  22 #include <linux/kernel_stat.h>
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/notifier.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/delay.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/tick.h>
  31 #include <linux/device.h>
  32 #include <linux/slab.h>
  33 #include <linux/cpu.h>
  34 #include <linux/completion.h>
  35 #include <linux/mutex.h>
  36 #include <linux/syscore_ops.h>
  37
  38 #include <trace/events/power.h>
  39
  40 /**
  41  * The "cpufreq driver" - the arch- or hardware-dependent low
  42  * level driver of CPUFreq support, and its spinlock. This lock
  43  * also protects the cpufreq_cpu_data array.
  44  */
  45 static struct cpufreq_driver *cpufreq_driver;
  46 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  47 static DEFINE_RWLOCK(cpufreq_driver_lock);
  48 static DEFINE_MUTEX(cpufreq_governor_lock);
  49
  50 #ifdef CONFIG_HOTPLUG_CPU
  51 /* This one keeps track of the previously set governor of a removed CPU */
  52 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  53 #endif
  54
  55 /*
  56  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  57  * all cpufreq/hotplug/workqueue/etc related lock issues.
  58  *
  59  * The rules for this semaphore:
  60  * - Any routine that wants to read from the policy structure will
  61  *   do a down_read on this semaphore.
  62  * - Any routine that will write to the policy structure and/or may take away
  63  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  64  *   mode before doing so.
  65  *
  66  * Additional rules:
  67  * - Governor routines that can be called in cpufreq hotplug path should not
  68  *   take this sem as top level hotplug notifier handler takes this.
  69  * - Lock should not be held across
  70  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  71  */
  72 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  73 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  74
  75 #define lock_policy_rwsem(mode, cpu)                                    \
  76 static int lock_policy_rwsem_##mode(int cpu)                            \
  77 {                                                                       \
  78         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  79         BUG_ON(policy_cpu == -1);                                       \
  80         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86 lock_policy_rwsem(write, cpu);
  87
  88 #define unlock_policy_rwsem(mode, cpu)                                  \
  89 static void unlock_policy_rwsem_##mode(int cpu)                         \
  90 {                                                                       \
  91         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  92         BUG_ON(policy_cpu == -1);                                       \
  93         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
  94 }
  95
  96 unlock_policy_rwsem(read, cpu);
  97 unlock_policy_rwsem(write, cpu);
  98
  99 /* internal prototypes */
 100 static int __cpufreq_governor(struct cpufreq_policy *policy,
 101                 unsigned int event);
 102 static unsigned int __cpufreq_get(unsigned int cpu);
 103 static void handle_update(struct work_struct *work);
 104
 105 /**
 106  * Two notifier lists: the "policy" list is involved in the
 107  * validation process for a new CPU frequency policy; the
 108  * "transition" list for kernel code that needs to handle
 109  * changes to devices when the CPU clock speed changes.
 110  * The mutex locks both lists.
 111  */
 112 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 113 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 114
 115 static bool init_cpufreq_transition_notifier_list_called;
 116 static int __init init_cpufreq_transition_notifier_list(void)
 117 {
 118         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 119         init_cpufreq_transition_notifier_list_called = true;
 120         return 0;
 121 }
 122 pure_initcall(init_cpufreq_transition_notifier_list);
 123
 124 static int off __read_mostly;
 125 static int cpufreq_disabled(void)
 126 {
 127         return off;
 128 }
 129 void disable_cpufreq(void)
 130 {
 131         off = 1;
 132 }
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 bool have_governor_per_policy(void)
 137 {
 138         return cpufreq_driver->have_governor_per_policy;
 139 }
 140 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 141
 142 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 143 {
 144         if (have_governor_per_policy())
 145                 return &policy->kobj;
 146         else
 147                 return cpufreq_global_kobject;
 148 }
 149 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 150
 151 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 152 {
 153         u64 idle_time;
 154         u64 cur_wall_time;
 155         u64 busy_time;
 156
 157         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 158
 159         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 165
 166         idle_time = cur_wall_time - busy_time;
 167         if (wall)
 168                 *wall = cputime_to_usecs(cur_wall_time);
 169
 170         return cputime_to_usecs(idle_time);
 171 }
 172
 173 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 174 {
 175         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 176
 177         if (idle_time == -1ULL)
 178                 return get_cpu_idle_time_jiffy(cpu, wall);
 179         else if (!io_busy)
 180                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 181
 182         return idle_time;
 183 }
 184 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 185
 186 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 187 {
 188         struct cpufreq_policy *data;
 189         unsigned long flags;
 190
 191         if (cpu >= nr_cpu_ids)
 192                 goto err_out;
 193
 194         /* get the cpufreq driver */
 195         read_lock_irqsave(&cpufreq_driver_lock, flags);
 196
 197         if (!cpufreq_driver)
 198                 goto err_out_unlock;
 199
 200         if (!try_module_get(cpufreq_driver->owner))
 201                 goto err_out_unlock;
 202
 203         /* get the CPU */
 204         data = per_cpu(cpufreq_cpu_data, cpu);
 205
 206         if (!data)
 207                 goto err_out_put_module;
 208
 209         if (!sysfs && !kobject_get(&data->kobj))
 210                 goto err_out_put_module;
 211
 212         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 213         return data;
 214
 215 err_out_put_module:
 216         module_put(cpufreq_driver->owner);
 217 err_out_unlock:
 218         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 219 err_out:
 220         return NULL;
 221 }
 222
 223 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 224 {
 225         if (cpufreq_disabled())
 226                 return NULL;
 227
 228         return __cpufreq_cpu_get(cpu, false);
 229 }
 230 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 231
 232 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 233 {
 234         return __cpufreq_cpu_get(cpu, true);
 235 }
 236
 237 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 238 {
 239         if (!sysfs)
 240                 kobject_put(&data->kobj);
 241         module_put(cpufreq_driver->owner);
 242 }
 243
 244 void cpufreq_cpu_put(struct cpufreq_policy *data)
 245 {
 246         if (cpufreq_disabled())
 247                 return;
 248
 249         __cpufreq_cpu_put(data, false);
 250 }
 251 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 252
 253 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 254 {
 255         __cpufreq_cpu_put(data, true);
 256 }
 257
 258 /*********************************************************************
 259  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 260  *********************************************************************/
 261
 262 /**
 263  * adjust_jiffies - adjust the system "loops_per_jiffy"
 264  *
 265  * This function alters the system "loops_per_jiffy" for the clock
 266  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 267  * systems as each CPU might be scaled differently. So, use the arch
 268  * per-CPU loops_per_jiffy value wherever possible.
 269  */
 270 #ifndef CONFIG_SMP
 271 static unsigned long l_p_j_ref;
 272 static unsigned int l_p_j_ref_freq;
 273
 274 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 275 {
 276         if (ci->flags & CPUFREQ_CONST_LOOPS)
 277                 return;
 278
 279         if (!l_p_j_ref_freq) {
 280                 l_p_j_ref = loops_per_jiffy;
 281                 l_p_j_ref_freq = ci->old;
 282                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 283                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 284         }
 285         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 286             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 287                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 288                                                                 ci->new);
 289                 pr_debug("scaling loops_per_jiffy to %lu "
 290                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 291         }
 292 }
 293 #else
 294 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 295 {
 296         return;
 297 }
 298 #endif
 299
 300 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 301                 struct cpufreq_freqs *freqs, unsigned int state)
 302 {
 303         BUG_ON(irqs_disabled());
 304
 305         if (cpufreq_disabled())
 306                 return;
 307
 308         freqs->flags = cpufreq_driver->flags;
 309         pr_debug("notification %u of frequency transition to %u kHz\n",
 310                 state, freqs->new);
 311
 312         switch (state) {
 313
 314         case CPUFREQ_PRECHANGE:
 315                 if (WARN(policy->transition_ongoing ==
 316                                         cpumask_weight(policy->cpus),
 317                                 "In middle of another frequency transition\n"))
 318                         return;
 319
 320                 policy->transition_ongoing++;
 321
 322                 /* detect if the driver reported a value as "old frequency"
 323                  * which is not equal to what the cpufreq core thinks is
 324                  * "old frequency".
 325                  */
 326                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 327                         if ((policy) && (policy->cpu == freqs->cpu) &&
 328                             (policy->cur) && (policy->cur != freqs->old)) {
 329                                 pr_debug("Warning: CPU frequency is"
 330                                         " %u, cpufreq assumed %u kHz.\n",
 331                                         freqs->old, policy->cur);
 332                                 freqs->old = policy->cur;
 333                         }
 334                 }
 335                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 336                                 CPUFREQ_PRECHANGE, freqs);
 337                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 338                 break;
 339
 340         case CPUFREQ_POSTCHANGE:
 341                 if (WARN(!policy->transition_ongoing,
 342                                 "No frequency transition in progress\n"))
 343                         return;
 344
 345                 policy->transition_ongoing--;
 346
 347                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 348                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 349                         (unsigned long)freqs->cpu);
 350                 trace_cpu_frequency(freqs->new, freqs->cpu);
 351                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 352                                 CPUFREQ_POSTCHANGE, freqs);
 353                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 354                         policy->cur = freqs->new;
 355                 break;
 356         }
 357 }
 358
 359 /**
 360  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 361  * on frequency transition.
 362  *
 363  * This function calls the transition notifiers and the "adjust_jiffies"
 364  * function. It is called twice on all CPU frequency changes that have
 365  * external effects.
 366  */
 367 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 368                 struct cpufreq_freqs *freqs, unsigned int state)
 369 {
 370         for_each_cpu(freqs->cpu, policy->cpus)
 371                 __cpufreq_notify_transition(policy, freqs, state);
 372 }
 373 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 374
 375
 376 /*********************************************************************
 377  *                          SYSFS INTERFACE                          *
 378  *********************************************************************/
 379
 380 static struct cpufreq_governor *__find_governor(const char *str_governor)
 381 {
 382         struct cpufreq_governor *t;
 383
 384         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 385                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 386                         return t;
 387
 388         return NULL;
 389 }
 390
 391 /**
 392  * cpufreq_parse_governor - parse a governor string
 393  */
 394 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 395                                 struct cpufreq_governor **governor)
 396 {
 397         int err = -EINVAL;
 398
 399         if (!cpufreq_driver)
 400                 goto out;
 401
 402         if (cpufreq_driver->setpolicy) {
 403                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 404                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 405                         err = 0;
 406                 } else if (!strnicmp(str_governor, "powersave",
 407                                                 CPUFREQ_NAME_LEN)) {
 408                         *policy = CPUFREQ_POLICY_POWERSAVE;
 409                         err = 0;
 410                 }
 411         } else if (cpufreq_driver->target) {
 412                 struct cpufreq_governor *t;
 413
 414                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                 t = __find_governor(str_governor);
 417
 418                 if (t == NULL) {
 419                         int ret;
 420
 421                         mutex_unlock(&cpufreq_governor_mutex);
 422                         ret = request_module("cpufreq_%s", str_governor);
 423                         mutex_lock(&cpufreq_governor_mutex);
 424
 425                         if (ret == 0)
 426                                 t = __find_governor(str_governor);
 427                 }
 428
 429                 if (t != NULL) {
 430                         *governor = t;
 431                         err = 0;
 432                 }
 433
 434                 mutex_unlock(&cpufreq_governor_mutex);
 435         }
 436 out:
 437         return err;
 438 }
 439
 440 /**
 441  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 442  * print out cpufreq information
 443  *
 444  * Write out information from cpufreq_driver->policy[cpu]; object must be
 445  * "unsigned int".
 446  */
 447
 448 #define show_one(file_name, object)                     \
 449 static ssize_t show_##file_name                         \
 450 (struct cpufreq_policy *policy, char *buf)              \
 451 {                                                       \
 452         return sprintf(buf, "%u\n", policy->object);    \
 453 }
 454
 455 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 456 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 457 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret;                                               \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf(buf, "%u", &new_policy.object);                    \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq, min);
 490 store_one(scaling_max_freq, max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504 /**
 505  * show_scaling_governor - show the current policy for the specified CPU
 506  */
 507 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 508 {
 509         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 515                                 policy->governor->name);
 516         return -EINVAL;
 517 }
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /*
 542          * Do not use cpufreq_set_policy here or the user_policy.max
 543          * will be wrongly overridden
 544          */
 545         ret = __cpufreq_set_policy(policy, &new_policy);
 546
 547         policy->user_policy.policy = policy->policy;
 548         policy->user_policy.governor = policy->governor;
 549
 550         if (ret)
 551                 return ret;
 552         else
 553                 return count;
 554 }
 555
 556 /**
 557  * show_scaling_driver - show the cpufreq driver currently loaded
 558  */
 559 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 560 {
 561         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 562 }
 563
 564 /**
 565  * show_scaling_available_governors - show the available CPUfreq governors
 566  */
 567 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 568                                                 char *buf)
 569 {
 570         ssize_t i = 0;
 571         struct cpufreq_governor *t;
 572
 573         if (!cpufreq_driver->target) {
 574                 i += sprintf(buf, "performance powersave");
 575                 goto out;
 576         }
 577
 578         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 579                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 580                     - (CPUFREQ_NAME_LEN + 2)))
 581                         goto out;
 582                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 583         }
 584 out:
 585         i += sprintf(&buf[i], "\n");
 586         return i;
 587 }
 588
 589 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 590 {
 591         ssize_t i = 0;
 592         unsigned int cpu;
 593
 594         for_each_cpu(cpu, mask) {
 595                 if (i)
 596                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 597                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 598                 if (i >= (PAGE_SIZE - 5))
 599                         break;
 600         }
 601         i += sprintf(&buf[i], "\n");
 602         return i;
 603 }
 604 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 605
 606 /**
 607  * show_related_cpus - show the CPUs affected by each transition even if
 608  * hw coordination is in use
 609  */
 610 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 611 {
 612         return cpufreq_show_cpus(policy->related_cpus, buf);
 613 }
 614
 615 /**
 616  * show_affected_cpus - show the CPUs affected by each transition
 617  */
 618 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 619 {
 620         return cpufreq_show_cpus(policy->cpus, buf);
 621 }
 622
 623 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 624                                         const char *buf, size_t count)
 625 {
 626         unsigned int freq = 0;
 627         unsigned int ret;
 628
 629         if (!policy->governor || !policy->governor->store_setspeed)
 630                 return -EINVAL;
 631
 632         ret = sscanf(buf, "%u", &freq);
 633         if (ret != 1)
 634                 return -EINVAL;
 635
 636         policy->governor->store_setspeed(policy, freq);
 637
 638         return count;
 639 }
 640
 641 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 642 {
 643         if (!policy->governor || !policy->governor->show_setspeed)
 644                 return sprintf(buf, "<unsupported>\n");
 645
 646         return policy->governor->show_setspeed(policy, buf);
 647 }
 648
 649 /**
 650  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 651  */
 652 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 653 {
 654         unsigned int limit;
 655         int ret;
 656         if (cpufreq_driver->bios_limit) {
 657                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 658                 if (!ret)
 659                         return sprintf(buf, "%u\n", limit);
 660         }
 661         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 662 }
 663
 664 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 665 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 666 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 667 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 668 cpufreq_freq_attr_ro(scaling_available_governors);
 669 cpufreq_freq_attr_ro(scaling_driver);
 670 cpufreq_freq_attr_ro(scaling_cur_freq);
 671 cpufreq_freq_attr_ro(bios_limit);
 672 cpufreq_freq_attr_ro(related_cpus);
 673 cpufreq_freq_attr_ro(affected_cpus);
 674 cpufreq_freq_attr_rw(scaling_min_freq);
 675 cpufreq_freq_attr_rw(scaling_max_freq);
 676 cpufreq_freq_attr_rw(scaling_governor);
 677 cpufreq_freq_attr_rw(scaling_setspeed);
 678
 679 static struct attribute *default_attrs[] = {
 680         &cpuinfo_min_freq.attr,
 681         &cpuinfo_max_freq.attr,
 682         &cpuinfo_transition_latency.attr,
 683         &scaling_min_freq.attr,
 684         &scaling_max_freq.attr,
 685         &affected_cpus.attr,
 686         &related_cpus.attr,
 687         &scaling_governor.attr,
 688         &scaling_driver.attr,
 689         &scaling_available_governors.attr,
 690         &scaling_setspeed.attr,
 691         NULL
 692 };
 693
 694 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 695 #define to_attr(a) container_of(a, struct freq_attr, attr)
 696
 697 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 698 {
 699         struct cpufreq_policy *policy = to_policy(kobj);
 700         struct freq_attr *fattr = to_attr(attr);
 701         ssize_t ret = -EINVAL;
 702         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 703         if (!policy)
 704                 goto no_policy;
 705
 706         if (lock_policy_rwsem_read(policy->cpu) < 0)
 707                 goto fail;
 708
 709         if (fattr->show)
 710                 ret = fattr->show(policy, buf);
 711         else
 712                 ret = -EIO;
 713
 714         unlock_policy_rwsem_read(policy->cpu);
 715 fail:
 716         cpufreq_cpu_put_sysfs(policy);
 717 no_policy:
 718         return ret;
 719 }
 720
 721 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 722                      const char *buf, size_t count)
 723 {
 724         struct cpufreq_policy *policy = to_policy(kobj);
 725         struct freq_attr *fattr = to_attr(attr);
 726         ssize_t ret = -EINVAL;
 727         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 728         if (!policy)
 729                 goto no_policy;
 730
 731         if (lock_policy_rwsem_write(policy->cpu) < 0)
 732                 goto fail;
 733
 734         if (fattr->store)
 735                 ret = fattr->store(policy, buf, count);
 736         else
 737                 ret = -EIO;
 738
 739         unlock_policy_rwsem_write(policy->cpu);
 740 fail:
 741         cpufreq_cpu_put_sysfs(policy);
 742 no_policy:
 743         return ret;
 744 }
 745
 746 static void cpufreq_sysfs_release(struct kobject *kobj)
 747 {
 748         struct cpufreq_policy *policy = to_policy(kobj);
 749         pr_debug("last reference is dropped\n");
 750         complete(&policy->kobj_unregister);
 751 }
 752
 753 static const struct sysfs_ops sysfs_ops = {
 754         .show   = show,
 755         .store  = store,
 756 };
 757
 758 static struct kobj_type ktype_cpufreq = {
 759         .sysfs_ops      = &sysfs_ops,
 760         .default_attrs  = default_attrs,
 761         .release        = cpufreq_sysfs_release,
 762 };
 763
 764 struct kobject *cpufreq_global_kobject;
 765 EXPORT_SYMBOL(cpufreq_global_kobject);
 766
 767 static int cpufreq_global_kobject_usage;
 768
 769 int cpufreq_get_global_kobject(void)
 770 {
 771         if (!cpufreq_global_kobject_usage++)
 772                 return kobject_add(cpufreq_global_kobject,
 773                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 774
 775         return 0;
 776 }
 777 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 778
 779 void cpufreq_put_global_kobject(void)
 780 {
 781         if (!--cpufreq_global_kobject_usage)
 782                 kobject_del(cpufreq_global_kobject);
 783 }
 784 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 785
 786 int cpufreq_sysfs_create_file(const struct attribute *attr)
 787 {
 788         int ret = cpufreq_get_global_kobject();
 789
 790         if (!ret) {
 791                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 792                 if (ret)
 793                         cpufreq_put_global_kobject();
 794         }
 795
 796         return ret;
 797 }
 798 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 799
 800 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 801 {
 802         sysfs_remove_file(cpufreq_global_kobject, attr);
 803         cpufreq_put_global_kobject();
 804 }
 805 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 806
 807 /* symlink affected CPUs */
 808 static int cpufreq_add_dev_symlink(unsigned int cpu,
 809                                    struct cpufreq_policy *policy)
 810 {
 811         unsigned int j;
 812         int ret = 0;
 813
 814         for_each_cpu(j, policy->cpus) {
 815                 struct cpufreq_policy *managed_policy;
 816                 struct device *cpu_dev;
 817
 818                 if (j == cpu)
 819                         continue;
 820
 821                 pr_debug("CPU %u already managed, adding link\n", j);
 822                 managed_policy = cpufreq_cpu_get(cpu);
 823                 cpu_dev = get_cpu_device(j);
 824                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 825                                         "cpufreq");
 826                 if (ret) {
 827                         cpufreq_cpu_put(managed_policy);
 828                         return ret;
 829                 }
 830         }
 831         return ret;
 832 }
 833
 834 static int cpufreq_add_dev_interface(unsigned int cpu,
 835                                      struct cpufreq_policy *policy,
 836                                      struct device *dev)
 837 {
 838         struct freq_attr **drv_attr;
 839         int ret = 0;
 840
 841         /* prepare interface data */
 842         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 843                                    &dev->kobj, "cpufreq");
 844         if (ret)
 845                 return ret;
 846
 847         /* set up files for this cpu device */
 848         drv_attr = cpufreq_driver->attr;
 849         while ((drv_attr) && (*drv_attr)) {
 850                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 851                 if (ret)
 852                         goto err_out_kobj_put;
 853                 drv_attr++;
 854         }
 855         if (cpufreq_driver->get) {
 856                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 857                 if (ret)
 858                         goto err_out_kobj_put;
 859         }
 860         if (cpufreq_driver->target) {
 861                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 862                 if (ret)
 863                         goto err_out_kobj_put;
 864         }
 865         if (cpufreq_driver->bios_limit) {
 866                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 867                 if (ret)
 868                         goto err_out_kobj_put;
 869         }
 870
 871         ret = cpufreq_add_dev_symlink(cpu, policy);
 872         if (ret)
 873                 goto err_out_kobj_put;
 874
 875         return ret;
 876
 877 err_out_kobj_put:
 878         kobject_put(&policy->kobj);
 879         wait_for_completion(&policy->kobj_unregister);
 880         return ret;
 881 }
 882
 883 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 884 {
 885         struct cpufreq_policy new_policy;
 886         int ret = 0;
 887
 888         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 889         /* assure that the starting sequence is run in __cpufreq_set_policy */
 890         policy->governor = NULL;
 891
 892         /* set default policy */
 893         ret = __cpufreq_set_policy(policy, &new_policy);
 894         policy->user_policy.policy = policy->policy;
 895         policy->user_policy.governor = policy->governor;
 896
 897         if (ret) {
 898                 pr_debug("setting policy failed\n");
 899                 if (cpufreq_driver->exit)
 900                         cpufreq_driver->exit(policy);
 901         }
 902 }
 903
 904 #ifdef CONFIG_HOTPLUG_CPU
 905 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 906                                   struct device *dev)
 907 {
 908         struct cpufreq_policy *policy;
 909         int ret = 0, has_target = !!cpufreq_driver->target;
 910         unsigned long flags;
 911
 912         policy = cpufreq_cpu_get(sibling);
 913         WARN_ON(!policy);
 914
 915         if (has_target)
 916                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 917
 918         lock_policy_rwsem_write(sibling);
 919
 920         write_lock_irqsave(&cpufreq_driver_lock, flags);
 921
 922         cpumask_set_cpu(cpu, policy->cpus);
 923         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 924         per_cpu(cpufreq_cpu_data, cpu) = policy;
 925         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 926
 927         unlock_policy_rwsem_write(sibling);
 928
 929         if (has_target) {
 930                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 931                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 932         }
 933
 934         ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 935         if (ret) {
 936                 cpufreq_cpu_put(policy);
 937                 return ret;
 938         }
 939
 940         return 0;
 941 }
 942 #endif
 943
 944 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 945 {
 946         struct cpufreq_policy *policy;
 947
 948         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 949         if (!policy)
 950                 return NULL;
 951
 952         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 953                 goto err_free_policy;
 954
 955         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 956                 goto err_free_cpumask;
 957
 958         return policy;
 959
 960 err_free_cpumask:
 961         free_cpumask_var(policy->cpus);
 962 err_free_policy:
 963         kfree(policy);
 964
 965         return NULL;
 966 }
 967
 968 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 969 {
 970         free_cpumask_var(policy->related_cpus);
 971         free_cpumask_var(policy->cpus);
 972         kfree(policy);
 973 }
 974
 975 /**
 976  * cpufreq_add_dev - add a CPU device
 977  *
 978  * Adds the cpufreq interface for a CPU device.
 979  *
 980  * The Oracle says: try running cpufreq registration/unregistration concurrently
 981  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 982  * mess up, but more thorough testing is needed. - Mathieu
 983  */
 984 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 985 {
 986         unsigned int j, cpu = dev->id;
 987         int ret = -ENOMEM;
 988         struct cpufreq_policy *policy;
 989         unsigned long flags;
 990 #ifdef CONFIG_HOTPLUG_CPU
 991         struct cpufreq_governor *gov;
 992         int sibling;
 993 #endif
 994
 995         if (cpu_is_offline(cpu))
 996                 return 0;
 997
 998         pr_debug("adding CPU %u\n", cpu);
 999
1000 #ifdef CONFIG_SMP
1001         /* check whether a different CPU already registered this
1002          * CPU because it is in the same boat. */
1003         policy = cpufreq_cpu_get(cpu);
1004         if (unlikely(policy)) {
1005                 cpufreq_cpu_put(policy);
1006                 return 0;
1007         }
1008
1009 #ifdef CONFIG_HOTPLUG_CPU
1010         /* Check if this cpu was hot-unplugged earlier and has siblings */
1011         read_lock_irqsave(&cpufreq_driver_lock, flags);
1012         for_each_online_cpu(sibling) {
1013                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
1014                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
1015                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1016                         return cpufreq_add_policy_cpu(cpu, sibling, dev);
1017                 }
1018         }
1019         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1020 #endif
1021 #endif
1022
1023         if (!try_module_get(cpufreq_driver->owner)) {
1024                 ret = -EINVAL;
1025                 goto module_out;
1026         }
1027
1028         policy = cpufreq_policy_alloc();
1029         if (!policy)
1030                 goto nomem_out;
1031
1032         policy->cpu = cpu;
1033         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1034         cpumask_copy(policy->cpus, cpumask_of(cpu));
1035
1036         /* Initially set CPU itself as the policy_cpu */
1037         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
1038
1039         init_completion(&policy->kobj_unregister);
1040         INIT_WORK(&policy->update, handle_update);
1041
1042         /* call driver. From then on the cpufreq must be able
1043          * to accept all calls to ->verify and ->setpolicy for this CPU
1044          */
1045         ret = cpufreq_driver->init(policy);
1046         if (ret) {
1047                 pr_debug("initialization failed\n");
1048                 goto err_set_policy_cpu;
1049         }
1050
1051         /* related cpus should atleast have policy->cpus */
1052         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1053
1054         /*
1055          * affected cpus must always be the one, which are online. We aren't
1056          * managing offline cpus here.
1057          */
1058         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1059
1060         policy->user_policy.min = policy->min;
1061         policy->user_policy.max = policy->max;
1062
1063         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1064                                      CPUFREQ_START, policy);
1065
1066 #ifdef CONFIG_HOTPLUG_CPU
1067         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1068         if (gov) {
1069                 policy->governor = gov;
1070                 pr_debug("Restoring governor %s for cpu %d\n",
1071                        policy->governor->name, cpu);
1072         }
1073 #endif
1074
1075         write_lock_irqsave(&cpufreq_driver_lock, flags);
1076         for_each_cpu(j, policy->cpus) {
1077                 per_cpu(cpufreq_cpu_data, j) = policy;
1078                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
1079         }
1080         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1081
1082         ret = cpufreq_add_dev_interface(cpu, policy, dev);
1083         if (ret)
1084                 goto err_out_unregister;
1085
1086         cpufreq_init_policy(policy);
1087
1088         kobject_uevent(&policy->kobj, KOBJ_ADD);
1089         module_put(cpufreq_driver->owner);
1090         pr_debug("initialization complete\n");
1091
1092         return 0;
1093
1094 err_out_unregister:
1095         write_lock_irqsave(&cpufreq_driver_lock, flags);
1096         for_each_cpu(j, policy->cpus) {
1097                 per_cpu(cpufreq_cpu_data, j) = NULL;
1098                 if (j != cpu)
1099                         per_cpu(cpufreq_policy_cpu, j) = -1;
1100         }
1101         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1102
1103         kobject_put(&policy->kobj);
1104         wait_for_completion(&policy->kobj_unregister);
1105
1106 err_set_policy_cpu:
1107         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1108         cpufreq_policy_free(policy);
1109 nomem_out:
1110         module_put(cpufreq_driver->owner);
1111 module_out:
1112         return ret;
1113 }
1114
1115 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1116 {
1117         int j;
1118
1119         policy->last_cpu = policy->cpu;
1120         policy->cpu = cpu;
1121
1122         for_each_cpu(j, policy->cpus)
1123                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1124
1125 #ifdef CONFIG_CPU_FREQ_TABLE
1126         cpufreq_frequency_table_update_policy_cpu(policy);
1127 #endif
1128         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1129                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1130 }
1131
1132 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *data,
1133                                            unsigned int old_cpu)
1134 {
1135         struct device *cpu_dev;
1136         unsigned long flags;
1137         int ret;
1138
1139         /* first sibling now owns the new sysfs dir */
1140         cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1141         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1142         ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1143         if (ret) {
1144                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1145
1146                 WARN_ON(lock_policy_rwsem_write(old_cpu));
1147                 cpumask_set_cpu(old_cpu, data->cpus);
1148
1149                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1150                 per_cpu(cpufreq_cpu_data, old_cpu) = data;
1151                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1152
1153                 unlock_policy_rwsem_write(old_cpu);
1154
1155                 ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1156                                         "cpufreq");
1157
1158                 return -EINVAL;
1159         }
1160
1161         return cpu_dev->id;
1162 }
1163
1164 /**
1165  * __cpufreq_remove_dev - remove a CPU device
1166  *
1167  * Removes the cpufreq interface for a CPU device.
1168  * Caller should already have policy_rwsem in write mode for this CPU.
1169  * This routine frees the rwsem before returning.
1170  */
1171 static int __cpufreq_remove_dev(struct device *dev,
1172                 struct subsys_interface *sif)
1173 {
1174         unsigned int cpu = dev->id, cpus;
1175         int new_cpu;
1176         unsigned long flags;
1177         struct cpufreq_policy *data;
1178         struct kobject *kobj;
1179         struct completion *cmp;
1180
1181         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1182
1183         write_lock_irqsave(&cpufreq_driver_lock, flags);
1184
1185         data = per_cpu(cpufreq_cpu_data, cpu);
1186         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1187
1188         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1189
1190         if (!data) {
1191                 pr_debug("%s: No cpu_data found\n", __func__);
1192                 return -EINVAL;
1193         }
1194
1195         if (cpufreq_driver->target)
1196                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1197
1198 #ifdef CONFIG_HOTPLUG_CPU
1199         if (!cpufreq_driver->setpolicy)
1200                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1201                         data->governor->name, CPUFREQ_NAME_LEN);
1202 #endif
1203
1204         WARN_ON(lock_policy_rwsem_write(cpu));
1205         cpus = cpumask_weight(data->cpus);
1206
1207         if (cpus > 1)
1208                 cpumask_clear_cpu(cpu, data->cpus);
1209         unlock_policy_rwsem_write(cpu);
1210
1211         if (cpu != data->cpu) {
1212                 sysfs_remove_link(&dev->kobj, "cpufreq");
1213         } else if (cpus > 1) {
1214
1215                 new_cpu = cpufreq_nominate_new_policy_cpu(data, cpu);
1216                 if (new_cpu >= 0) {
1217                         WARN_ON(lock_policy_rwsem_write(cpu));
1218                         update_policy_cpu(data, new_cpu);
1219                         unlock_policy_rwsem_write(cpu);
1220                         pr_debug("%s: policy Kobject moved to cpu: %d "
1221                                  "from: %d\n",__func__, new_cpu, cpu);
1222                 }
1223         }
1224
1225         /* If cpu is last user of policy, free policy */
1226         if (cpus == 1) {
1227                 if (cpufreq_driver->target)
1228                         __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1229
1230                 lock_policy_rwsem_read(cpu);
1231                 kobj = &data->kobj;
1232                 cmp = &data->kobj_unregister;
1233                 unlock_policy_rwsem_read(cpu);
1234                 kobject_put(kobj);
1235
1236                 /* we need to make sure that the underlying kobj is actually
1237                  * not referenced anymore by anybody before we proceed with
1238                  * unloading.
1239                  */
1240                 pr_debug("waiting for dropping of refcount\n");
1241                 wait_for_completion(cmp);
1242                 pr_debug("wait complete\n");
1243
1244                 if (cpufreq_driver->exit)
1245                         cpufreq_driver->exit(data);
1246
1247                 cpufreq_policy_free(data);
1248         } else {
1249                 pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1250                 cpufreq_cpu_put(data);
1251                 if (cpufreq_driver->target) {
1252                         __cpufreq_governor(data, CPUFREQ_GOV_START);
1253                         __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1254                 }
1255         }
1256
1257         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1258         return 0;
1259 }
1260
1261 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1262 {
1263         unsigned int cpu = dev->id;
1264         int retval;
1265
1266         if (cpu_is_offline(cpu))
1267                 return 0;
1268
1269         retval = __cpufreq_remove_dev(dev, sif);
1270         return retval;
1271 }
1272
1273 static void handle_update(struct work_struct *work)
1274 {
1275         struct cpufreq_policy *policy =
1276                 container_of(work, struct cpufreq_policy, update);
1277         unsigned int cpu = policy->cpu;
1278         pr_debug("handle_update for cpu %u called\n", cpu);
1279         cpufreq_update_policy(cpu);
1280 }
1281
1282 /**
1283  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1284  *      in deep trouble.
1285  *      @cpu: cpu number
1286  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1287  *      @new_freq: CPU frequency the CPU actually runs at
1288  *
1289  *      We adjust to current frequency first, and need to clean up later.
1290  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1291  */
1292 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1293                                 unsigned int new_freq)
1294 {
1295         struct cpufreq_policy *policy;
1296         struct cpufreq_freqs freqs;
1297         unsigned long flags;
1298
1299         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1300                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1301
1302         freqs.old = old_freq;
1303         freqs.new = new_freq;
1304
1305         read_lock_irqsave(&cpufreq_driver_lock, flags);
1306         policy = per_cpu(cpufreq_cpu_data, cpu);
1307         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1308
1309         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1310         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1311 }
1312
1313 /**
1314  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1315  * @cpu: CPU number
1316  *
1317  * This is the last known freq, without actually getting it from the driver.
1318  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1319  */
1320 unsigned int cpufreq_quick_get(unsigned int cpu)
1321 {
1322         struct cpufreq_policy *policy;
1323         unsigned int ret_freq = 0;
1324
1325         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1326                 return cpufreq_driver->get(cpu);
1327
1328         policy = cpufreq_cpu_get(cpu);
1329         if (policy) {
1330                 ret_freq = policy->cur;
1331                 cpufreq_cpu_put(policy);
1332         }
1333
1334         return ret_freq;
1335 }
1336 EXPORT_SYMBOL(cpufreq_quick_get);
1337
1338 /**
1339  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1340  * @cpu: CPU number
1341  *
1342  * Just return the max possible frequency for a given CPU.
1343  */
1344 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1345 {
1346         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1347         unsigned int ret_freq = 0;
1348
1349         if (policy) {
1350                 ret_freq = policy->max;
1351                 cpufreq_cpu_put(policy);
1352         }
1353
1354         return ret_freq;
1355 }
1356 EXPORT_SYMBOL(cpufreq_quick_get_max);
1357
1358 static unsigned int __cpufreq_get(unsigned int cpu)
1359 {
1360         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1361         unsigned int ret_freq = 0;
1362
1363         if (!cpufreq_driver->get)
1364                 return ret_freq;
1365
1366         ret_freq = cpufreq_driver->get(cpu);
1367
1368         if (ret_freq && policy->cur &&
1369                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1370                 /* verify no discrepancy between actual and
1371                                         saved value exists */
1372                 if (unlikely(ret_freq != policy->cur)) {
1373                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1374                         schedule_work(&policy->update);
1375                 }
1376         }
1377
1378         return ret_freq;
1379 }
1380
1381 /**
1382  * cpufreq_get - get the current CPU frequency (in kHz)
1383  * @cpu: CPU number
1384  *
1385  * Get the CPU current (static) CPU frequency
1386  */
1387 unsigned int cpufreq_get(unsigned int cpu)
1388 {
1389         unsigned int ret_freq = 0;
1390         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1391
1392         if (!policy)
1393                 goto out;
1394
1395         if (unlikely(lock_policy_rwsem_read(cpu)))
1396                 goto out_policy;
1397
1398         ret_freq = __cpufreq_get(cpu);
1399
1400         unlock_policy_rwsem_read(cpu);
1401
1402 out_policy:
1403         cpufreq_cpu_put(policy);
1404 out:
1405         return ret_freq;
1406 }
1407 EXPORT_SYMBOL(cpufreq_get);
1408
1409 static struct subsys_interface cpufreq_interface = {
1410         .name           = "cpufreq",
1411         .subsys         = &cpu_subsys,
1412         .add_dev        = cpufreq_add_dev,
1413         .remove_dev     = cpufreq_remove_dev,
1414 };
1415
1416 /**
1417  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1418  *
1419  * This function is only executed for the boot processor.  The other CPUs
1420  * have been put offline by means of CPU hotplug.
1421  */
1422 static int cpufreq_bp_suspend(void)
1423 {
1424         int ret = 0;
1425
1426         int cpu = smp_processor_id();
1427         struct cpufreq_policy *cpu_policy;
1428
1429         pr_debug("suspending cpu %u\n", cpu);
1430
1431         /* If there's no policy for the boot CPU, we have nothing to do. */
1432         cpu_policy = cpufreq_cpu_get(cpu);
1433         if (!cpu_policy)
1434                 return 0;
1435
1436         if (cpufreq_driver->suspend) {
1437                 ret = cpufreq_driver->suspend(cpu_policy);
1438                 if (ret)
1439                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1440                                         "step on CPU %u\n", cpu_policy->cpu);
1441         }
1442
1443         cpufreq_cpu_put(cpu_policy);
1444         return ret;
1445 }
1446
1447 /**
1448  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1449  *
1450  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1451  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1452  *          restored. It will verify that the current freq is in sync with
1453  *          what we believe it to be. This is a bit later than when it
1454  *          should be, but nonethteless it's better than calling
1455  *          cpufreq_driver->get() here which might re-enable interrupts...
1456  *
1457  * This function is only executed for the boot CPU.  The other CPUs have not
1458  * been turned on yet.
1459  */
1460 static void cpufreq_bp_resume(void)
1461 {
1462         int ret = 0;
1463
1464         int cpu = smp_processor_id();
1465         struct cpufreq_policy *cpu_policy;
1466
1467         pr_debug("resuming cpu %u\n", cpu);
1468
1469         /* If there's no policy for the boot CPU, we have nothing to do. */
1470         cpu_policy = cpufreq_cpu_get(cpu);
1471         if (!cpu_policy)
1472                 return;
1473
1474         if (cpufreq_driver->resume) {
1475                 ret = cpufreq_driver->resume(cpu_policy);
1476                 if (ret) {
1477                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1478                                         "step on CPU %u\n", cpu_policy->cpu);
1479                         goto fail;
1480                 }
1481         }
1482
1483         schedule_work(&cpu_policy->update);
1484
1485 fail:
1486         cpufreq_cpu_put(cpu_policy);
1487 }
1488
1489 static struct syscore_ops cpufreq_syscore_ops = {
1490         .suspend        = cpufreq_bp_suspend,
1491         .resume         = cpufreq_bp_resume,
1492 };
1493
1494 /**
1495  *      cpufreq_get_current_driver - return current driver's name
1496  *
1497  *      Return the name string of the currently loaded cpufreq driver
1498  *      or NULL, if none.
1499  */
1500 const char *cpufreq_get_current_driver(void)
1501 {
1502         if (cpufreq_driver)
1503                 return cpufreq_driver->name;
1504
1505         return NULL;
1506 }
1507 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1508
1509 /*********************************************************************
1510  *                     NOTIFIER LISTS INTERFACE                      *
1511  *********************************************************************/
1512
1513 /**
1514  *      cpufreq_register_notifier - register a driver with cpufreq
1515  *      @nb: notifier function to register
1516  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1517  *
1518  *      Add a driver to one of two lists: either a list of drivers that
1519  *      are notified about clock rate changes (once before and once after
1520  *      the transition), or a list of drivers that are notified about
1521  *      changes in cpufreq policy.
1522  *
1523  *      This function may sleep, and has the same return conditions as
1524  *      blocking_notifier_chain_register.
1525  */
1526 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1527 {
1528         int ret;
1529
1530         if (cpufreq_disabled())
1531                 return -EINVAL;
1532
1533         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1534
1535         switch (list) {
1536         case CPUFREQ_TRANSITION_NOTIFIER:
1537                 ret = srcu_notifier_chain_register(
1538                                 &cpufreq_transition_notifier_list, nb);
1539                 break;
1540         case CPUFREQ_POLICY_NOTIFIER:
1541                 ret = blocking_notifier_chain_register(
1542                                 &cpufreq_policy_notifier_list, nb);
1543                 break;
1544         default:
1545                 ret = -EINVAL;
1546         }
1547
1548         return ret;
1549 }
1550 EXPORT_SYMBOL(cpufreq_register_notifier);
1551
1552 /**
1553  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1554  *      @nb: notifier block to be unregistered
1555  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1556  *
1557  *      Remove a driver from the CPU frequency notifier list.
1558  *
1559  *      This function may sleep, and has the same return conditions as
1560  *      blocking_notifier_chain_unregister.
1561  */
1562 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1563 {
1564         int ret;
1565
1566         if (cpufreq_disabled())
1567                 return -EINVAL;
1568
1569         switch (list) {
1570         case CPUFREQ_TRANSITION_NOTIFIER:
1571                 ret = srcu_notifier_chain_unregister(
1572                                 &cpufreq_transition_notifier_list, nb);
1573                 break;
1574         case CPUFREQ_POLICY_NOTIFIER:
1575                 ret = blocking_notifier_chain_unregister(
1576                                 &cpufreq_policy_notifier_list, nb);
1577                 break;
1578         default:
1579                 ret = -EINVAL;
1580         }
1581
1582         return ret;
1583 }
1584 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1585
1586
1587 /*********************************************************************
1588  *                              GOVERNORS                            *
1589  *********************************************************************/
1590
1591 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1592                             unsigned int target_freq,
1593                             unsigned int relation)
1594 {
1595         int retval = -EINVAL;
1596         unsigned int old_target_freq = target_freq;
1597
1598         if (cpufreq_disabled())
1599                 return -ENODEV;
1600         if (policy->transition_ongoing)
1601                 return -EBUSY;
1602
1603         /* Make sure that target_freq is within supported range */
1604         if (target_freq > policy->max)
1605                 target_freq = policy->max;
1606         if (target_freq < policy->min)
1607                 target_freq = policy->min;
1608
1609         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1610                         policy->cpu, target_freq, relation, old_target_freq);
1611
1612         if (target_freq == policy->cur)
1613                 return 0;
1614
1615         if (cpufreq_driver->target)
1616                 retval = cpufreq_driver->target(policy, target_freq, relation);
1617
1618         return retval;
1619 }
1620 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1621
1622 int cpufreq_driver_target(struct cpufreq_policy *policy,
1623                           unsigned int target_freq,
1624                           unsigned int relation)
1625 {
1626         int ret = -EINVAL;
1627
1628         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1629                 goto fail;
1630
1631         ret = __cpufreq_driver_target(policy, target_freq, relation);
1632
1633         unlock_policy_rwsem_write(policy->cpu);
1634
1635 fail:
1636         return ret;
1637 }
1638 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1639
1640 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1641 {
1642         if (cpufreq_disabled())
1643                 return 0;
1644
1645         if (!cpufreq_driver->getavg)
1646                 return 0;
1647
1648         return cpufreq_driver->getavg(policy, cpu);
1649 }
1650 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1651
1652 /*
1653  * when "event" is CPUFREQ_GOV_LIMITS
1654  */
1655
1656 static int __cpufreq_governor(struct cpufreq_policy *policy,
1657                                         unsigned int event)
1658 {
1659         int ret;
1660
1661         /* Only must be defined when default governor is known to have latency
1662            restrictions, like e.g. conservative or ondemand.
1663            That this is the case is already ensured in Kconfig
1664         */
1665 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1666         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1667 #else
1668         struct cpufreq_governor *gov = NULL;
1669 #endif
1670
1671         if (policy->governor->max_transition_latency &&
1672             policy->cpuinfo.transition_latency >
1673             policy->governor->max_transition_latency) {
1674                 if (!gov)
1675                         return -EINVAL;
1676                 else {
1677                         printk(KERN_WARNING "%s governor failed, too long"
1678                                " transition latency of HW, fallback"
1679                                " to %s governor\n",
1680                                policy->governor->name,
1681                                gov->name);
1682                         policy->governor = gov;
1683                 }
1684         }
1685
1686         if (!try_module_get(policy->governor->owner))
1687                 return -EINVAL;
1688
1689         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1690                                                 policy->cpu, event);
1691
1692         mutex_lock(&cpufreq_governor_lock);
1693         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1694             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1695                 mutex_unlock(&cpufreq_governor_lock);
1696                 return -EBUSY;
1697         }
1698
1699         if (event == CPUFREQ_GOV_STOP)
1700                 policy->governor_enabled = false;
1701         else if (event == CPUFREQ_GOV_START)
1702                 policy->governor_enabled = true;
1703
1704         mutex_unlock(&cpufreq_governor_lock);
1705
1706         ret = policy->governor->governor(policy, event);
1707
1708         if (!ret) {
1709                 if (event == CPUFREQ_GOV_POLICY_INIT)
1710                         policy->governor->initialized++;
1711                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1712                         policy->governor->initialized--;
1713         } else {
1714                 /* Restore original values */
1715                 mutex_lock(&cpufreq_governor_lock);
1716                 if (event == CPUFREQ_GOV_STOP)
1717                         policy->governor_enabled = true;
1718                 else if (event == CPUFREQ_GOV_START)
1719                         policy->governor_enabled = false;
1720                 mutex_unlock(&cpufreq_governor_lock);
1721         }
1722
1723         /* we keep one module reference alive for
1724                         each CPU governed by this CPU */
1725         if ((event != CPUFREQ_GOV_START) || ret)
1726                 module_put(policy->governor->owner);
1727         if ((event == CPUFREQ_GOV_STOP) && !ret)
1728                 module_put(policy->governor->owner);
1729
1730         return ret;
1731 }
1732
1733 int cpufreq_register_governor(struct cpufreq_governor *governor)
1734 {
1735         int err;
1736
1737         if (!governor)
1738                 return -EINVAL;
1739
1740         if (cpufreq_disabled())
1741                 return -ENODEV;
1742
1743         mutex_lock(&cpufreq_governor_mutex);
1744
1745         governor->initialized = 0;
1746         err = -EBUSY;
1747         if (__find_governor(governor->name) == NULL) {
1748                 err = 0;
1749                 list_add(&governor->governor_list, &cpufreq_governor_list);
1750         }
1751
1752         mutex_unlock(&cpufreq_governor_mutex);
1753         return err;
1754 }
1755 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1756
1757 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1758 {
1759 #ifdef CONFIG_HOTPLUG_CPU
1760         int cpu;
1761 #endif
1762
1763         if (!governor)
1764                 return;
1765
1766         if (cpufreq_disabled())
1767                 return;
1768
1769 #ifdef CONFIG_HOTPLUG_CPU
1770         for_each_present_cpu(cpu) {
1771                 if (cpu_online(cpu))
1772                         continue;
1773                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1774                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1775         }
1776 #endif
1777
1778         mutex_lock(&cpufreq_governor_mutex);
1779         list_del(&governor->governor_list);
1780         mutex_unlock(&cpufreq_governor_mutex);
1781         return;
1782 }
1783 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1784
1785
1786 /*********************************************************************
1787  *                          POLICY INTERFACE                         *
1788  *********************************************************************/
1789
1790 /**
1791  * cpufreq_get_policy - get the current cpufreq_policy
1792  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1793  *      is written
1794  *
1795  * Reads the current cpufreq policy.
1796  */
1797 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1798 {
1799         struct cpufreq_policy *cpu_policy;
1800         if (!policy)
1801                 return -EINVAL;
1802
1803         cpu_policy = cpufreq_cpu_get(cpu);
1804         if (!cpu_policy)
1805                 return -EINVAL;
1806
1807         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1808
1809         cpufreq_cpu_put(cpu_policy);
1810         return 0;
1811 }
1812 EXPORT_SYMBOL(cpufreq_get_policy);
1813
1814 /*
1815  * data   : current policy.
1816  * policy : policy to be set.
1817  */
1818 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1819                                 struct cpufreq_policy *policy)
1820 {
1821         int ret = 0, failed = 1;
1822
1823         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1824                 policy->min, policy->max);
1825
1826         memcpy(&policy->cpuinfo, &data->cpuinfo,
1827                                 sizeof(struct cpufreq_cpuinfo));
1828
1829         if (policy->min > data->max || policy->max < data->min) {
1830                 ret = -EINVAL;
1831                 goto error_out;
1832         }
1833
1834         /* verify the cpu speed can be set within this limit */
1835         ret = cpufreq_driver->verify(policy);
1836         if (ret)
1837                 goto error_out;
1838
1839         /* adjust if necessary - all reasons */
1840         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1841                         CPUFREQ_ADJUST, policy);
1842
1843         /* adjust if necessary - hardware incompatibility*/
1844         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1845                         CPUFREQ_INCOMPATIBLE, policy);
1846
1847         /*
1848          * verify the cpu speed can be set within this limit, which might be
1849          * different to the first one
1850          */
1851         ret = cpufreq_driver->verify(policy);
1852         if (ret)
1853                 goto error_out;
1854
1855         /* notification of the new policy */
1856         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1857                         CPUFREQ_NOTIFY, policy);
1858
1859         data->min = policy->min;
1860         data->max = policy->max;
1861
1862         pr_debug("new min and max freqs are %u - %u kHz\n",
1863                                         data->min, data->max);
1864
1865         if (cpufreq_driver->setpolicy) {
1866                 data->policy = policy->policy;
1867                 pr_debug("setting range\n");
1868                 ret = cpufreq_driver->setpolicy(policy);
1869         } else {
1870                 if (policy->governor != data->governor) {
1871                         /* save old, working values */
1872                         struct cpufreq_governor *old_gov = data->governor;
1873
1874                         pr_debug("governor switch\n");
1875
1876                         /* end old governor */
1877                         if (data->governor) {
1878                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1879                                 unlock_policy_rwsem_write(policy->cpu);
1880                                 __cpufreq_governor(data,
1881                                                 CPUFREQ_GOV_POLICY_EXIT);
1882                                 lock_policy_rwsem_write(policy->cpu);
1883                         }
1884
1885                         /* start new governor */
1886                         data->governor = policy->governor;
1887                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1888                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1889                                         failed = 0;
1890                                 } else {
1891                                         unlock_policy_rwsem_write(policy->cpu);
1892                                         __cpufreq_governor(data,
1893                                                         CPUFREQ_GOV_POLICY_EXIT);
1894                                         lock_policy_rwsem_write(policy->cpu);
1895                                 }
1896                         }
1897
1898                         if (failed) {
1899                                 /* new governor failed, so re-start old one */
1900                                 pr_debug("starting governor %s failed\n",
1901                                                         data->governor->name);
1902                                 if (old_gov) {
1903                                         data->governor = old_gov;
1904                                         __cpufreq_governor(data,
1905                                                         CPUFREQ_GOV_POLICY_INIT);
1906                                         __cpufreq_governor(data,
1907                                                            CPUFREQ_GOV_START);
1908                                 }
1909                                 ret = -EINVAL;
1910                                 goto error_out;
1911                         }
1912                         /* might be a policy change, too, so fall through */
1913                 }
1914                 pr_debug("governor: change or update limits\n");
1915                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1916         }
1917
1918 error_out:
1919         return ret;
1920 }
1921
1922 /**
1923  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1924  *      @cpu: CPU which shall be re-evaluated
1925  *
1926  *      Useful for policy notifiers which have different necessities
1927  *      at different times.
1928  */
1929 int cpufreq_update_policy(unsigned int cpu)
1930 {
1931         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1932         struct cpufreq_policy policy;
1933         int ret;
1934
1935         if (!data) {
1936                 ret = -ENODEV;
1937                 goto no_policy;
1938         }
1939
1940         if (unlikely(lock_policy_rwsem_write(cpu))) {
1941                 ret = -EINVAL;
1942                 goto fail;
1943         }
1944
1945         pr_debug("updating policy for CPU %u\n", cpu);
1946         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1947         policy.min = data->user_policy.min;
1948         policy.max = data->user_policy.max;
1949         policy.policy = data->user_policy.policy;
1950         policy.governor = data->user_policy.governor;
1951
1952         /*
1953          * BIOS might change freq behind our back
1954          * -> ask driver for current freq and notify governors about a change
1955          */
1956         if (cpufreq_driver->get) {
1957                 policy.cur = cpufreq_driver->get(cpu);
1958                 if (!data->cur) {
1959                         pr_debug("Driver did not initialize current freq");
1960                         data->cur = policy.cur;
1961                 } else {
1962                         if (data->cur != policy.cur && cpufreq_driver->target)
1963                                 cpufreq_out_of_sync(cpu, data->cur,
1964                                                                 policy.cur);
1965                 }
1966         }
1967
1968         ret = __cpufreq_set_policy(data, &policy);
1969
1970         unlock_policy_rwsem_write(cpu);
1971
1972 fail:
1973         cpufreq_cpu_put(data);
1974 no_policy:
1975         return ret;
1976 }
1977 EXPORT_SYMBOL(cpufreq_update_policy);
1978
1979 static int cpufreq_cpu_callback(struct notifier_block *nfb,
1980                                         unsigned long action, void *hcpu)
1981 {
1982         unsigned int cpu = (unsigned long)hcpu;
1983         struct device *dev;
1984
1985         dev = get_cpu_device(cpu);
1986         if (dev) {
1987                 switch (action) {
1988                 case CPU_ONLINE:
1989                 case CPU_ONLINE_FROZEN:
1990                         cpufreq_add_dev(dev, NULL);
1991                         cpufreq_update_policy(cpu);
1992                         break;
1993                 case CPU_DOWN_PREPARE:
1994                 case CPU_DOWN_PREPARE_FROZEN:
1995                         __cpufreq_remove_dev(dev, NULL);
1996                         break;
1997                 case CPU_DOWN_FAILED:
1998                 case CPU_DOWN_FAILED_FROZEN:
1999                         cpufreq_add_dev(dev, NULL);
2000                         break;
2001                 }
2002         }
2003         return NOTIFY_OK;
2004 }
2005
2006 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2007         .notifier_call = cpufreq_cpu_callback,
2008 };
2009
2010 /*********************************************************************
2011  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2012  *********************************************************************/
2013
2014 /**
2015  * cpufreq_register_driver - register a CPU Frequency driver
2016  * @driver_data: A struct cpufreq_driver containing the values#
2017  * submitted by the CPU Frequency driver.
2018  *
2019  * Registers a CPU Frequency driver to this core code. This code
2020  * returns zero on success, -EBUSY when another driver got here first
2021  * (and isn't unregistered in the meantime).
2022  *
2023  */
2024 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2025 {
2026         unsigned long flags;
2027         int ret;
2028
2029         if (cpufreq_disabled())
2030                 return -ENODEV;
2031
2032         if (!driver_data || !driver_data->verify || !driver_data->init ||
2033             ((!driver_data->setpolicy) && (!driver_data->target)))
2034                 return -EINVAL;
2035
2036         pr_debug("trying to register driver %s\n", driver_data->name);
2037
2038         if (driver_data->setpolicy)
2039                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2040
2041         write_lock_irqsave(&cpufreq_driver_lock, flags);
2042         if (cpufreq_driver) {
2043                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2044                 return -EBUSY;
2045         }
2046         cpufreq_driver = driver_data;
2047         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2048
2049         ret = subsys_interface_register(&cpufreq_interface);
2050         if (ret)
2051                 goto err_null_driver;
2052
2053         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2054                 int i;
2055                 ret = -ENODEV;
2056
2057                 /* check for at least one working CPU */
2058                 for (i = 0; i < nr_cpu_ids; i++)
2059                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2060                                 ret = 0;
2061                                 break;
2062                         }
2063
2064                 /* if all ->init() calls failed, unregister */
2065                 if (ret) {
2066                         pr_debug("no CPU initialized for driver %s\n",
2067                                                         driver_data->name);
2068                         goto err_if_unreg;
2069                 }
2070         }
2071
2072         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2073         pr_debug("driver %s up and running\n", driver_data->name);
2074
2075         return 0;
2076 err_if_unreg:
2077         subsys_interface_unregister(&cpufreq_interface);
2078 err_null_driver:
2079         write_lock_irqsave(&cpufreq_driver_lock, flags);
2080         cpufreq_driver = NULL;
2081         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2082         return ret;
2083 }
2084 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2085
2086 /**
2087  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2088  *
2089  * Unregister the current CPUFreq driver. Only call this if you have
2090  * the right to do so, i.e. if you have succeeded in initialising before!
2091  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2092  * currently not initialised.
2093  */
2094 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2095 {
2096         unsigned long flags;
2097
2098         if (!cpufreq_driver || (driver != cpufreq_driver))
2099                 return -EINVAL;
2100
2101         pr_debug("unregistering driver %s\n", driver->name);
2102
2103         subsys_interface_unregister(&cpufreq_interface);
2104         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2105
2106         write_lock_irqsave(&cpufreq_driver_lock, flags);
2107         cpufreq_driver = NULL;
2108         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2109
2110         return 0;
2111 }
2112 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2113
2114 static int __init cpufreq_core_init(void)
2115 {
2116         int cpu;
2117
2118         if (cpufreq_disabled())
2119                 return -ENODEV;
2120
2121         for_each_possible_cpu(cpu) {
2122                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2123                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2124         }
2125
2126         cpufreq_global_kobject = kobject_create();
2127         BUG_ON(!cpufreq_global_kobject);
2128         register_syscore_ops(&cpufreq_syscore_ops);
2129
2130         return 0;
2131 }
2132 core_initcall(cpufreq_core_init);