Pileus Git - ~andy/linux/blob - drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <asm/cputime.h>
  21 #include <linux/kernel.h>
  22 #include <linux/kernel_stat.h>
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/notifier.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/delay.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/tick.h>
  31 #include <linux/device.h>
  32 #include <linux/slab.h>
  33 #include <linux/cpu.h>
  34 #include <linux/completion.h>
  35 #include <linux/mutex.h>
  36 #include <linux/syscore_ops.h>
  37
  38 #include <trace/events/power.h>
  39
  40 /**
  41  * The "cpufreq driver" - the arch- or hardware-dependent low
  42  * level driver of CPUFreq support, and its spinlock. This lock
  43  * also protects the cpufreq_cpu_data array.
  44  */
  45 static struct cpufreq_driver *cpufreq_driver;
  46 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  47 static DEFINE_RWLOCK(cpufreq_driver_lock);
  48 static DEFINE_MUTEX(cpufreq_governor_lock);
  49
  50 #ifdef CONFIG_HOTPLUG_CPU
  51 /* This one keeps track of the previously set governor of a removed CPU */
  52 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  53 #endif
  54
  55 /*
  56  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  57  * all cpufreq/hotplug/workqueue/etc related lock issues.
  58  *
  59  * The rules for this semaphore:
  60  * - Any routine that wants to read from the policy structure will
  61  *   do a down_read on this semaphore.
  62  * - Any routine that will write to the policy structure and/or may take away
  63  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  64  *   mode before doing so.
  65  *
  66  * Additional rules:
  67  * - Governor routines that can be called in cpufreq hotplug path should not
  68  *   take this sem as top level hotplug notifier handler takes this.
  69  * - Lock should not be held across
  70  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  71  */
  72 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  73 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  74
  75 #define lock_policy_rwsem(mode, cpu)                                    \
  76 static int lock_policy_rwsem_##mode(int cpu)                            \
  77 {                                                                       \
  78         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  79         BUG_ON(policy_cpu == -1);                                       \
  80         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86 lock_policy_rwsem(write, cpu);
  87
  88 #define unlock_policy_rwsem(mode, cpu)                                  \
  89 static void unlock_policy_rwsem_##mode(int cpu)                         \
  90 {                                                                       \
  91         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  92         BUG_ON(policy_cpu == -1);                                       \
  93         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
  94 }
  95
  96 unlock_policy_rwsem(read, cpu);
  97 unlock_policy_rwsem(write, cpu);
  98
  99 /* internal prototypes */
 100 static int __cpufreq_governor(struct cpufreq_policy *policy,
 101                 unsigned int event);
 102 static unsigned int __cpufreq_get(unsigned int cpu);
 103 static void handle_update(struct work_struct *work);
 104
 105 /**
 106  * Two notifier lists: the "policy" list is involved in the
 107  * validation process for a new CPU frequency policy; the
 108  * "transition" list for kernel code that needs to handle
 109  * changes to devices when the CPU clock speed changes.
 110  * The mutex locks both lists.
 111  */
 112 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 113 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 114
 115 static bool init_cpufreq_transition_notifier_list_called;
 116 static int __init init_cpufreq_transition_notifier_list(void)
 117 {
 118         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 119         init_cpufreq_transition_notifier_list_called = true;
 120         return 0;
 121 }
 122 pure_initcall(init_cpufreq_transition_notifier_list);
 123
 124 static int off __read_mostly;
 125 static int cpufreq_disabled(void)
 126 {
 127         return off;
 128 }
 129 void disable_cpufreq(void)
 130 {
 131         off = 1;
 132 }
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 bool have_governor_per_policy(void)
 137 {
 138         return cpufreq_driver->have_governor_per_policy;
 139 }
 140 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 141
 142 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 143 {
 144         if (have_governor_per_policy())
 145                 return &policy->kobj;
 146         else
 147                 return cpufreq_global_kobject;
 148 }
 149 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 150
 151 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 152 {
 153         u64 idle_time;
 154         u64 cur_wall_time;
 155         u64 busy_time;
 156
 157         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 158
 159         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 165
 166         idle_time = cur_wall_time - busy_time;
 167         if (wall)
 168                 *wall = cputime_to_usecs(cur_wall_time);
 169
 170         return cputime_to_usecs(idle_time);
 171 }
 172
 173 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 174 {
 175         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 176
 177         if (idle_time == -1ULL)
 178                 return get_cpu_idle_time_jiffy(cpu, wall);
 179         else if (!io_busy)
 180                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 181
 182         return idle_time;
 183 }
 184 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 185
 186 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 187 {
 188         struct cpufreq_policy *data;
 189         unsigned long flags;
 190
 191         if (cpu >= nr_cpu_ids)
 192                 goto err_out;
 193
 194         /* get the cpufreq driver */
 195         read_lock_irqsave(&cpufreq_driver_lock, flags);
 196
 197         if (!cpufreq_driver)
 198                 goto err_out_unlock;
 199
 200         if (!try_module_get(cpufreq_driver->owner))
 201                 goto err_out_unlock;
 202
 203         /* get the CPU */
 204         data = per_cpu(cpufreq_cpu_data, cpu);
 205
 206         if (!data)
 207                 goto err_out_put_module;
 208
 209         if (!sysfs && !kobject_get(&data->kobj))
 210                 goto err_out_put_module;
 211
 212         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 213         return data;
 214
 215 err_out_put_module:
 216         module_put(cpufreq_driver->owner);
 217 err_out_unlock:
 218         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 219 err_out:
 220         return NULL;
 221 }
 222
 223 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 224 {
 225         if (cpufreq_disabled())
 226                 return NULL;
 227
 228         return __cpufreq_cpu_get(cpu, false);
 229 }
 230 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 231
 232 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 233 {
 234         return __cpufreq_cpu_get(cpu, true);
 235 }
 236
 237 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 238 {
 239         if (!sysfs)
 240                 kobject_put(&data->kobj);
 241         module_put(cpufreq_driver->owner);
 242 }
 243
 244 void cpufreq_cpu_put(struct cpufreq_policy *data)
 245 {
 246         if (cpufreq_disabled())
 247                 return;
 248
 249         __cpufreq_cpu_put(data, false);
 250 }
 251 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 252
 253 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 254 {
 255         __cpufreq_cpu_put(data, true);
 256 }
 257
 258 /*********************************************************************
 259  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 260  *********************************************************************/
 261
 262 /**
 263  * adjust_jiffies - adjust the system "loops_per_jiffy"
 264  *
 265  * This function alters the system "loops_per_jiffy" for the clock
 266  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 267  * systems as each CPU might be scaled differently. So, use the arch
 268  * per-CPU loops_per_jiffy value wherever possible.
 269  */
 270 #ifndef CONFIG_SMP
 271 static unsigned long l_p_j_ref;
 272 static unsigned int l_p_j_ref_freq;
 273
 274 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 275 {
 276         if (ci->flags & CPUFREQ_CONST_LOOPS)
 277                 return;
 278
 279         if (!l_p_j_ref_freq) {
 280                 l_p_j_ref = loops_per_jiffy;
 281                 l_p_j_ref_freq = ci->old;
 282                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 283                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 284         }
 285         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 286             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 287                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 288                                                                 ci->new);
 289                 pr_debug("scaling loops_per_jiffy to %lu "
 290                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 291         }
 292 }
 293 #else
 294 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 295 {
 296         return;
 297 }
 298 #endif
 299
 300 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 301                 struct cpufreq_freqs *freqs, unsigned int state)
 302 {
 303         BUG_ON(irqs_disabled());
 304
 305         if (cpufreq_disabled())
 306                 return;
 307
 308         freqs->flags = cpufreq_driver->flags;
 309         pr_debug("notification %u of frequency transition to %u kHz\n",
 310                 state, freqs->new);
 311
 312         switch (state) {
 313
 314         case CPUFREQ_PRECHANGE:
 315                 if (WARN(policy->transition_ongoing ==
 316                                         cpumask_weight(policy->cpus),
 317                                 "In middle of another frequency transition\n"))
 318                         return;
 319
 320                 policy->transition_ongoing++;
 321
 322                 /* detect if the driver reported a value as "old frequency"
 323                  * which is not equal to what the cpufreq core thinks is
 324                  * "old frequency".
 325                  */
 326                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 327                         if ((policy) && (policy->cpu == freqs->cpu) &&
 328                             (policy->cur) && (policy->cur != freqs->old)) {
 329                                 pr_debug("Warning: CPU frequency is"
 330                                         " %u, cpufreq assumed %u kHz.\n",
 331                                         freqs->old, policy->cur);
 332                                 freqs->old = policy->cur;
 333                         }
 334                 }
 335                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 336                                 CPUFREQ_PRECHANGE, freqs);
 337                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 338                 break;
 339
 340         case CPUFREQ_POSTCHANGE:
 341                 if (WARN(!policy->transition_ongoing,
 342                                 "No frequency transition in progress\n"))
 343                         return;
 344
 345                 policy->transition_ongoing--;
 346
 347                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 348                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 349                         (unsigned long)freqs->cpu);
 350                 trace_cpu_frequency(freqs->new, freqs->cpu);
 351                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 352                                 CPUFREQ_POSTCHANGE, freqs);
 353                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 354                         policy->cur = freqs->new;
 355                 break;
 356         }
 357 }
 358
 359 /**
 360  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 361  * on frequency transition.
 362  *
 363  * This function calls the transition notifiers and the "adjust_jiffies"
 364  * function. It is called twice on all CPU frequency changes that have
 365  * external effects.
 366  */
 367 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 368                 struct cpufreq_freqs *freqs, unsigned int state)
 369 {
 370         for_each_cpu(freqs->cpu, policy->cpus)
 371                 __cpufreq_notify_transition(policy, freqs, state);
 372 }
 373 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 374
 375
 376 /*********************************************************************
 377  *                          SYSFS INTERFACE                          *
 378  *********************************************************************/
 379
 380 static struct cpufreq_governor *__find_governor(const char *str_governor)
 381 {
 382         struct cpufreq_governor *t;
 383
 384         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 385                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 386                         return t;
 387
 388         return NULL;
 389 }
 390
 391 /**
 392  * cpufreq_parse_governor - parse a governor string
 393  */
 394 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 395                                 struct cpufreq_governor **governor)
 396 {
 397         int err = -EINVAL;
 398
 399         if (!cpufreq_driver)
 400                 goto out;
 401
 402         if (cpufreq_driver->setpolicy) {
 403                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 404                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 405                         err = 0;
 406                 } else if (!strnicmp(str_governor, "powersave",
 407                                                 CPUFREQ_NAME_LEN)) {
 408                         *policy = CPUFREQ_POLICY_POWERSAVE;
 409                         err = 0;
 410                 }
 411         } else if (cpufreq_driver->target) {
 412                 struct cpufreq_governor *t;
 413
 414                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                 t = __find_governor(str_governor);
 417
 418                 if (t == NULL) {
 419                         int ret;
 420
 421                         mutex_unlock(&cpufreq_governor_mutex);
 422                         ret = request_module("cpufreq_%s", str_governor);
 423                         mutex_lock(&cpufreq_governor_mutex);
 424
 425                         if (ret == 0)
 426                                 t = __find_governor(str_governor);
 427                 }
 428
 429                 if (t != NULL) {
 430                         *governor = t;
 431                         err = 0;
 432                 }
 433
 434                 mutex_unlock(&cpufreq_governor_mutex);
 435         }
 436 out:
 437         return err;
 438 }
 439
 440 /**
 441  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 442  * print out cpufreq information
 443  *
 444  * Write out information from cpufreq_driver->policy[cpu]; object must be
 445  * "unsigned int".
 446  */
 447
 448 #define show_one(file_name, object)                     \
 449 static ssize_t show_##file_name                         \
 450 (struct cpufreq_policy *policy, char *buf)              \
 451 {                                                       \
 452         return sprintf(buf, "%u\n", policy->object);    \
 453 }
 454
 455 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 456 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 457 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret;                                               \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf(buf, "%u", &new_policy.object);                    \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq, min);
 490 store_one(scaling_max_freq, max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504 /**
 505  * show_scaling_governor - show the current policy for the specified CPU
 506  */
 507 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 508 {
 509         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 515                                 policy->governor->name);
 516         return -EINVAL;
 517 }
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /*
 542          * Do not use cpufreq_set_policy here or the user_policy.max
 543          * will be wrongly overridden
 544          */
 545         ret = __cpufreq_set_policy(policy, &new_policy);
 546
 547         policy->user_policy.policy = policy->policy;
 548         policy->user_policy.governor = policy->governor;
 549
 550         if (ret)
 551                 return ret;
 552         else
 553                 return count;
 554 }
 555
 556 /**
 557  * show_scaling_driver - show the cpufreq driver currently loaded
 558  */
 559 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 560 {
 561         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 562 }
 563
 564 /**
 565  * show_scaling_available_governors - show the available CPUfreq governors
 566  */
 567 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 568                                                 char *buf)
 569 {
 570         ssize_t i = 0;
 571         struct cpufreq_governor *t;
 572
 573         if (!cpufreq_driver->target) {
 574                 i += sprintf(buf, "performance powersave");
 575                 goto out;
 576         }
 577
 578         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 579                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 580                     - (CPUFREQ_NAME_LEN + 2)))
 581                         goto out;
 582                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 583         }
 584 out:
 585         i += sprintf(&buf[i], "\n");
 586         return i;
 587 }
 588
 589 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 590 {
 591         ssize_t i = 0;
 592         unsigned int cpu;
 593
 594         for_each_cpu(cpu, mask) {
 595                 if (i)
 596                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 597                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 598                 if (i >= (PAGE_SIZE - 5))
 599                         break;
 600         }
 601         i += sprintf(&buf[i], "\n");
 602         return i;
 603 }
 604 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 605
 606 /**
 607  * show_related_cpus - show the CPUs affected by each transition even if
 608  * hw coordination is in use
 609  */
 610 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 611 {
 612         return cpufreq_show_cpus(policy->related_cpus, buf);
 613 }
 614
 615 /**
 616  * show_affected_cpus - show the CPUs affected by each transition
 617  */
 618 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 619 {
 620         return cpufreq_show_cpus(policy->cpus, buf);
 621 }
 622
 623 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 624                                         const char *buf, size_t count)
 625 {
 626         unsigned int freq = 0;
 627         unsigned int ret;
 628
 629         if (!policy->governor || !policy->governor->store_setspeed)
 630                 return -EINVAL;
 631
 632         ret = sscanf(buf, "%u", &freq);
 633         if (ret != 1)
 634                 return -EINVAL;
 635
 636         policy->governor->store_setspeed(policy, freq);
 637
 638         return count;
 639 }
 640
 641 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 642 {
 643         if (!policy->governor || !policy->governor->show_setspeed)
 644                 return sprintf(buf, "<unsupported>\n");
 645
 646         return policy->governor->show_setspeed(policy, buf);
 647 }
 648
 649 /**
 650  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 651  */
 652 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 653 {
 654         unsigned int limit;
 655         int ret;
 656         if (cpufreq_driver->bios_limit) {
 657                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 658                 if (!ret)
 659                         return sprintf(buf, "%u\n", limit);
 660         }
 661         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 662 }
 663
 664 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 665 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 666 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 667 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 668 cpufreq_freq_attr_ro(scaling_available_governors);
 669 cpufreq_freq_attr_ro(scaling_driver);
 670 cpufreq_freq_attr_ro(scaling_cur_freq);
 671 cpufreq_freq_attr_ro(bios_limit);
 672 cpufreq_freq_attr_ro(related_cpus);
 673 cpufreq_freq_attr_ro(affected_cpus);
 674 cpufreq_freq_attr_rw(scaling_min_freq);
 675 cpufreq_freq_attr_rw(scaling_max_freq);
 676 cpufreq_freq_attr_rw(scaling_governor);
 677 cpufreq_freq_attr_rw(scaling_setspeed);
 678
 679 static struct attribute *default_attrs[] = {
 680         &cpuinfo_min_freq.attr,
 681         &cpuinfo_max_freq.attr,
 682         &cpuinfo_transition_latency.attr,
 683         &scaling_min_freq.attr,
 684         &scaling_max_freq.attr,
 685         &affected_cpus.attr,
 686         &related_cpus.attr,
 687         &scaling_governor.attr,
 688         &scaling_driver.attr,
 689         &scaling_available_governors.attr,
 690         &scaling_setspeed.attr,
 691         NULL
 692 };
 693
 694 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 695 #define to_attr(a) container_of(a, struct freq_attr, attr)
 696
 697 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 698 {
 699         struct cpufreq_policy *policy = to_policy(kobj);
 700         struct freq_attr *fattr = to_attr(attr);
 701         ssize_t ret = -EINVAL;
 702         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 703         if (!policy)
 704                 goto no_policy;
 705
 706         if (lock_policy_rwsem_read(policy->cpu) < 0)
 707                 goto fail;
 708
 709         if (fattr->show)
 710                 ret = fattr->show(policy, buf);
 711         else
 712                 ret = -EIO;
 713
 714         unlock_policy_rwsem_read(policy->cpu);
 715 fail:
 716         cpufreq_cpu_put_sysfs(policy);
 717 no_policy:
 718         return ret;
 719 }
 720
 721 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 722                      const char *buf, size_t count)
 723 {
 724         struct cpufreq_policy *policy = to_policy(kobj);
 725         struct freq_attr *fattr = to_attr(attr);
 726         ssize_t ret = -EINVAL;
 727         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 728         if (!policy)
 729                 goto no_policy;
 730
 731         if (lock_policy_rwsem_write(policy->cpu) < 0)
 732                 goto fail;
 733
 734         if (fattr->store)
 735                 ret = fattr->store(policy, buf, count);
 736         else
 737                 ret = -EIO;
 738
 739         unlock_policy_rwsem_write(policy->cpu);
 740 fail:
 741         cpufreq_cpu_put_sysfs(policy);
 742 no_policy:
 743         return ret;
 744 }
 745
 746 static void cpufreq_sysfs_release(struct kobject *kobj)
 747 {
 748         struct cpufreq_policy *policy = to_policy(kobj);
 749         pr_debug("last reference is dropped\n");
 750         complete(&policy->kobj_unregister);
 751 }
 752
 753 static const struct sysfs_ops sysfs_ops = {
 754         .show   = show,
 755         .store  = store,
 756 };
 757
 758 static struct kobj_type ktype_cpufreq = {
 759         .sysfs_ops      = &sysfs_ops,
 760         .default_attrs  = default_attrs,
 761         .release        = cpufreq_sysfs_release,
 762 };
 763
 764 struct kobject *cpufreq_global_kobject;
 765 EXPORT_SYMBOL(cpufreq_global_kobject);
 766
 767 static int cpufreq_global_kobject_usage;
 768
 769 int cpufreq_get_global_kobject(void)
 770 {
 771         if (!cpufreq_global_kobject_usage++)
 772                 return kobject_add(cpufreq_global_kobject,
 773                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 774
 775         return 0;
 776 }
 777 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 778
 779 void cpufreq_put_global_kobject(void)
 780 {
 781         if (!--cpufreq_global_kobject_usage)
 782                 kobject_del(cpufreq_global_kobject);
 783 }
 784 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 785
 786 int cpufreq_sysfs_create_file(const struct attribute *attr)
 787 {
 788         int ret = cpufreq_get_global_kobject();
 789
 790         if (!ret) {
 791                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 792                 if (ret)
 793                         cpufreq_put_global_kobject();
 794         }
 795
 796         return ret;
 797 }
 798 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 799
 800 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 801 {
 802         sysfs_remove_file(cpufreq_global_kobject, attr);
 803         cpufreq_put_global_kobject();
 804 }
 805 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 806
 807 /* symlink affected CPUs */
 808 static int cpufreq_add_dev_symlink(unsigned int cpu,
 809                                    struct cpufreq_policy *policy)
 810 {
 811         unsigned int j;
 812         int ret = 0;
 813
 814         for_each_cpu(j, policy->cpus) {
 815                 struct cpufreq_policy *managed_policy;
 816                 struct device *cpu_dev;
 817
 818                 if (j == cpu)
 819                         continue;
 820
 821                 pr_debug("CPU %u already managed, adding link\n", j);
 822                 managed_policy = cpufreq_cpu_get(cpu);
 823                 cpu_dev = get_cpu_device(j);
 824                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 825                                         "cpufreq");
 826                 if (ret) {
 827                         cpufreq_cpu_put(managed_policy);
 828                         return ret;
 829                 }
 830         }
 831         return ret;
 832 }
 833
 834 static int cpufreq_add_dev_interface(unsigned int cpu,
 835                                      struct cpufreq_policy *policy,
 836                                      struct device *dev)
 837 {
 838         struct cpufreq_policy new_policy;
 839         struct freq_attr **drv_attr;
 840         unsigned long flags;
 841         int ret = 0;
 842         unsigned int j;
 843
 844         /* prepare interface data */
 845         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 846                                    &dev->kobj, "cpufreq");
 847         if (ret)
 848                 return ret;
 849
 850         /* set up files for this cpu device */
 851         drv_attr = cpufreq_driver->attr;
 852         while ((drv_attr) && (*drv_attr)) {
 853                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 854                 if (ret)
 855                         goto err_out_kobj_put;
 856                 drv_attr++;
 857         }
 858         if (cpufreq_driver->get) {
 859                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 860                 if (ret)
 861                         goto err_out_kobj_put;
 862         }
 863         if (cpufreq_driver->target) {
 864                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 865                 if (ret)
 866                         goto err_out_kobj_put;
 867         }
 868         if (cpufreq_driver->bios_limit) {
 869                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 870                 if (ret)
 871                         goto err_out_kobj_put;
 872         }
 873
 874         write_lock_irqsave(&cpufreq_driver_lock, flags);
 875         for_each_cpu(j, policy->cpus) {
 876                 per_cpu(cpufreq_cpu_data, j) = policy;
 877                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 878         }
 879         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 880
 881         ret = cpufreq_add_dev_symlink(cpu, policy);
 882         if (ret)
 883                 goto err_out_kobj_put;
 884
 885         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 886         /* assure that the starting sequence is run in __cpufreq_set_policy */
 887         policy->governor = NULL;
 888
 889         /* set default policy */
 890         ret = __cpufreq_set_policy(policy, &new_policy);
 891         policy->user_policy.policy = policy->policy;
 892         policy->user_policy.governor = policy->governor;
 893
 894         if (ret) {
 895                 pr_debug("setting policy failed\n");
 896                 if (cpufreq_driver->exit)
 897                         cpufreq_driver->exit(policy);
 898         }
 899         return ret;
 900
 901 err_out_kobj_put:
 902         kobject_put(&policy->kobj);
 903         wait_for_completion(&policy->kobj_unregister);
 904         return ret;
 905 }
 906
 907 #ifdef CONFIG_HOTPLUG_CPU
 908 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 909                                   struct device *dev)
 910 {
 911         struct cpufreq_policy *policy;
 912         int ret = 0, has_target = !!cpufreq_driver->target;
 913         unsigned long flags;
 914
 915         policy = cpufreq_cpu_get(sibling);
 916         WARN_ON(!policy);
 917
 918         if (has_target)
 919                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 920
 921         lock_policy_rwsem_write(sibling);
 922
 923         write_lock_irqsave(&cpufreq_driver_lock, flags);
 924
 925         cpumask_set_cpu(cpu, policy->cpus);
 926         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 927         per_cpu(cpufreq_cpu_data, cpu) = policy;
 928         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 929
 930         unlock_policy_rwsem_write(sibling);
 931
 932         if (has_target) {
 933                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 934                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 935         }
 936
 937         ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 938         if (ret) {
 939                 cpufreq_cpu_put(policy);
 940                 return ret;
 941         }
 942
 943         return 0;
 944 }
 945 #endif
 946
 947 /**
 948  * cpufreq_add_dev - add a CPU device
 949  *
 950  * Adds the cpufreq interface for a CPU device.
 951  *
 952  * The Oracle says: try running cpufreq registration/unregistration concurrently
 953  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 954  * mess up, but more thorough testing is needed. - Mathieu
 955  */
 956 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 957 {
 958         unsigned int j, cpu = dev->id;
 959         int ret = -ENOMEM;
 960         struct cpufreq_policy *policy;
 961         unsigned long flags;
 962 #ifdef CONFIG_HOTPLUG_CPU
 963         struct cpufreq_governor *gov;
 964         int sibling;
 965 #endif
 966
 967         if (cpu_is_offline(cpu))
 968                 return 0;
 969
 970         pr_debug("adding CPU %u\n", cpu);
 971
 972 #ifdef CONFIG_SMP
 973         /* check whether a different CPU already registered this
 974          * CPU because it is in the same boat. */
 975         policy = cpufreq_cpu_get(cpu);
 976         if (unlikely(policy)) {
 977                 cpufreq_cpu_put(policy);
 978                 return 0;
 979         }
 980
 981 #ifdef CONFIG_HOTPLUG_CPU
 982         /* Check if this cpu was hot-unplugged earlier and has siblings */
 983         read_lock_irqsave(&cpufreq_driver_lock, flags);
 984         for_each_online_cpu(sibling) {
 985                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 986                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
 987                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 988                         return cpufreq_add_policy_cpu(cpu, sibling, dev);
 989                 }
 990         }
 991         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 992 #endif
 993 #endif
 994
 995         if (!try_module_get(cpufreq_driver->owner)) {
 996                 ret = -EINVAL;
 997                 goto module_out;
 998         }
 999
1000         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
1001         if (!policy)
1002                 goto nomem_out;
1003
1004         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1005                 goto err_free_policy;
1006
1007         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1008                 goto err_free_cpumask;
1009
1010         policy->cpu = cpu;
1011         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1012         cpumask_copy(policy->cpus, cpumask_of(cpu));
1013
1014         /* Initially set CPU itself as the policy_cpu */
1015         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
1016
1017         init_completion(&policy->kobj_unregister);
1018         INIT_WORK(&policy->update, handle_update);
1019
1020         /* call driver. From then on the cpufreq must be able
1021          * to accept all calls to ->verify and ->setpolicy for this CPU
1022          */
1023         ret = cpufreq_driver->init(policy);
1024         if (ret) {
1025                 pr_debug("initialization failed\n");
1026                 goto err_set_policy_cpu;
1027         }
1028
1029         /* related cpus should atleast have policy->cpus */
1030         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1031
1032         /*
1033          * affected cpus must always be the one, which are online. We aren't
1034          * managing offline cpus here.
1035          */
1036         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1037
1038         policy->user_policy.min = policy->min;
1039         policy->user_policy.max = policy->max;
1040
1041         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1042                                      CPUFREQ_START, policy);
1043
1044 #ifdef CONFIG_HOTPLUG_CPU
1045         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1046         if (gov) {
1047                 policy->governor = gov;
1048                 pr_debug("Restoring governor %s for cpu %d\n",
1049                        policy->governor->name, cpu);
1050         }
1051 #endif
1052
1053         ret = cpufreq_add_dev_interface(cpu, policy, dev);
1054         if (ret)
1055                 goto err_out_unregister;
1056
1057         kobject_uevent(&policy->kobj, KOBJ_ADD);
1058         module_put(cpufreq_driver->owner);
1059         pr_debug("initialization complete\n");
1060
1061         return 0;
1062
1063 err_out_unregister:
1064         write_lock_irqsave(&cpufreq_driver_lock, flags);
1065         for_each_cpu(j, policy->cpus)
1066                 per_cpu(cpufreq_cpu_data, j) = NULL;
1067         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1068
1069         kobject_put(&policy->kobj);
1070         wait_for_completion(&policy->kobj_unregister);
1071
1072 err_set_policy_cpu:
1073         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1074         free_cpumask_var(policy->related_cpus);
1075 err_free_cpumask:
1076         free_cpumask_var(policy->cpus);
1077 err_free_policy:
1078         kfree(policy);
1079 nomem_out:
1080         module_put(cpufreq_driver->owner);
1081 module_out:
1082         return ret;
1083 }
1084
1085 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1086 {
1087         int j;
1088
1089         policy->last_cpu = policy->cpu;
1090         policy->cpu = cpu;
1091
1092         for_each_cpu(j, policy->cpus)
1093                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1094
1095 #ifdef CONFIG_CPU_FREQ_TABLE
1096         cpufreq_frequency_table_update_policy_cpu(policy);
1097 #endif
1098         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1099                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1100 }
1101
1102 /**
1103  * __cpufreq_remove_dev - remove a CPU device
1104  *
1105  * Removes the cpufreq interface for a CPU device.
1106  * Caller should already have policy_rwsem in write mode for this CPU.
1107  * This routine frees the rwsem before returning.
1108  */
1109 static int __cpufreq_remove_dev(struct device *dev,
1110                 struct subsys_interface *sif)
1111 {
1112         unsigned int cpu = dev->id, ret, cpus;
1113         unsigned long flags;
1114         struct cpufreq_policy *data;
1115         struct kobject *kobj;
1116         struct completion *cmp;
1117         struct device *cpu_dev;
1118
1119         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1120
1121         write_lock_irqsave(&cpufreq_driver_lock, flags);
1122
1123         data = per_cpu(cpufreq_cpu_data, cpu);
1124         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1125
1126         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1127
1128         if (!data) {
1129                 pr_debug("%s: No cpu_data found\n", __func__);
1130                 return -EINVAL;
1131         }
1132
1133         if (cpufreq_driver->target)
1134                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1135
1136 #ifdef CONFIG_HOTPLUG_CPU
1137         if (!cpufreq_driver->setpolicy)
1138                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1139                         data->governor->name, CPUFREQ_NAME_LEN);
1140 #endif
1141
1142         WARN_ON(lock_policy_rwsem_write(cpu));
1143         cpus = cpumask_weight(data->cpus);
1144
1145         if (cpus > 1)
1146                 cpumask_clear_cpu(cpu, data->cpus);
1147         unlock_policy_rwsem_write(cpu);
1148
1149         if (cpu != data->cpu) {
1150                 sysfs_remove_link(&dev->kobj, "cpufreq");
1151         } else if (cpus > 1) {
1152                 /* first sibling now owns the new sysfs dir */
1153                 cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1154                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1155                 ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1156                 if (ret) {
1157                         pr_err("%s: Failed to move kobj: %d", __func__, ret);
1158
1159                         WARN_ON(lock_policy_rwsem_write(cpu));
1160                         cpumask_set_cpu(cpu, data->cpus);
1161
1162                         write_lock_irqsave(&cpufreq_driver_lock, flags);
1163                         per_cpu(cpufreq_cpu_data, cpu) = data;
1164                         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1165
1166                         unlock_policy_rwsem_write(cpu);
1167
1168                         ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1169                                         "cpufreq");
1170                         return -EINVAL;
1171                 }
1172
1173                 WARN_ON(lock_policy_rwsem_write(cpu));
1174                 update_policy_cpu(data, cpu_dev->id);
1175                 unlock_policy_rwsem_write(cpu);
1176                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1177                                 __func__, cpu_dev->id, cpu);
1178         }
1179
1180         if ((cpus == 1) && (cpufreq_driver->target))
1181                 __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1182
1183         pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1184         cpufreq_cpu_put(data);
1185
1186         /* If cpu is last user of policy, free policy */
1187         if (cpus == 1) {
1188                 lock_policy_rwsem_read(cpu);
1189                 kobj = &data->kobj;
1190                 cmp = &data->kobj_unregister;
1191                 unlock_policy_rwsem_read(cpu);
1192                 kobject_put(kobj);
1193
1194                 /* we need to make sure that the underlying kobj is actually
1195                  * not referenced anymore by anybody before we proceed with
1196                  * unloading.
1197                  */
1198                 pr_debug("waiting for dropping of refcount\n");
1199                 wait_for_completion(cmp);
1200                 pr_debug("wait complete\n");
1201
1202                 if (cpufreq_driver->exit)
1203                         cpufreq_driver->exit(data);
1204
1205                 free_cpumask_var(data->related_cpus);
1206                 free_cpumask_var(data->cpus);
1207                 kfree(data);
1208         } else if (cpufreq_driver->target) {
1209                 __cpufreq_governor(data, CPUFREQ_GOV_START);
1210                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1211         }
1212
1213         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1214         return 0;
1215 }
1216
1217 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1218 {
1219         unsigned int cpu = dev->id;
1220         int retval;
1221
1222         if (cpu_is_offline(cpu))
1223                 return 0;
1224
1225         retval = __cpufreq_remove_dev(dev, sif);
1226         return retval;
1227 }
1228
1229 static void handle_update(struct work_struct *work)
1230 {
1231         struct cpufreq_policy *policy =
1232                 container_of(work, struct cpufreq_policy, update);
1233         unsigned int cpu = policy->cpu;
1234         pr_debug("handle_update for cpu %u called\n", cpu);
1235         cpufreq_update_policy(cpu);
1236 }
1237
1238 /**
1239  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1240  *      in deep trouble.
1241  *      @cpu: cpu number
1242  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1243  *      @new_freq: CPU frequency the CPU actually runs at
1244  *
1245  *      We adjust to current frequency first, and need to clean up later.
1246  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1247  */
1248 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1249                                 unsigned int new_freq)
1250 {
1251         struct cpufreq_policy *policy;
1252         struct cpufreq_freqs freqs;
1253         unsigned long flags;
1254
1255         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1256                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1257
1258         freqs.old = old_freq;
1259         freqs.new = new_freq;
1260
1261         read_lock_irqsave(&cpufreq_driver_lock, flags);
1262         policy = per_cpu(cpufreq_cpu_data, cpu);
1263         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1264
1265         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1266         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1267 }
1268
1269 /**
1270  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1271  * @cpu: CPU number
1272  *
1273  * This is the last known freq, without actually getting it from the driver.
1274  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1275  */
1276 unsigned int cpufreq_quick_get(unsigned int cpu)
1277 {
1278         struct cpufreq_policy *policy;
1279         unsigned int ret_freq = 0;
1280
1281         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1282                 return cpufreq_driver->get(cpu);
1283
1284         policy = cpufreq_cpu_get(cpu);
1285         if (policy) {
1286                 ret_freq = policy->cur;
1287                 cpufreq_cpu_put(policy);
1288         }
1289
1290         return ret_freq;
1291 }
1292 EXPORT_SYMBOL(cpufreq_quick_get);
1293
1294 /**
1295  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1296  * @cpu: CPU number
1297  *
1298  * Just return the max possible frequency for a given CPU.
1299  */
1300 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1301 {
1302         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1303         unsigned int ret_freq = 0;
1304
1305         if (policy) {
1306                 ret_freq = policy->max;
1307                 cpufreq_cpu_put(policy);
1308         }
1309
1310         return ret_freq;
1311 }
1312 EXPORT_SYMBOL(cpufreq_quick_get_max);
1313
1314 static unsigned int __cpufreq_get(unsigned int cpu)
1315 {
1316         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1317         unsigned int ret_freq = 0;
1318
1319         if (!cpufreq_driver->get)
1320                 return ret_freq;
1321
1322         ret_freq = cpufreq_driver->get(cpu);
1323
1324         if (ret_freq && policy->cur &&
1325                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1326                 /* verify no discrepancy between actual and
1327                                         saved value exists */
1328                 if (unlikely(ret_freq != policy->cur)) {
1329                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1330                         schedule_work(&policy->update);
1331                 }
1332         }
1333
1334         return ret_freq;
1335 }
1336
1337 /**
1338  * cpufreq_get - get the current CPU frequency (in kHz)
1339  * @cpu: CPU number
1340  *
1341  * Get the CPU current (static) CPU frequency
1342  */
1343 unsigned int cpufreq_get(unsigned int cpu)
1344 {
1345         unsigned int ret_freq = 0;
1346         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1347
1348         if (!policy)
1349                 goto out;
1350
1351         if (unlikely(lock_policy_rwsem_read(cpu)))
1352                 goto out_policy;
1353
1354         ret_freq = __cpufreq_get(cpu);
1355
1356         unlock_policy_rwsem_read(cpu);
1357
1358 out_policy:
1359         cpufreq_cpu_put(policy);
1360 out:
1361         return ret_freq;
1362 }
1363 EXPORT_SYMBOL(cpufreq_get);
1364
1365 static struct subsys_interface cpufreq_interface = {
1366         .name           = "cpufreq",
1367         .subsys         = &cpu_subsys,
1368         .add_dev        = cpufreq_add_dev,
1369         .remove_dev     = cpufreq_remove_dev,
1370 };
1371
1372 /**
1373  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1374  *
1375  * This function is only executed for the boot processor.  The other CPUs
1376  * have been put offline by means of CPU hotplug.
1377  */
1378 static int cpufreq_bp_suspend(void)
1379 {
1380         int ret = 0;
1381
1382         int cpu = smp_processor_id();
1383         struct cpufreq_policy *cpu_policy;
1384
1385         pr_debug("suspending cpu %u\n", cpu);
1386
1387         /* If there's no policy for the boot CPU, we have nothing to do. */
1388         cpu_policy = cpufreq_cpu_get(cpu);
1389         if (!cpu_policy)
1390                 return 0;
1391
1392         if (cpufreq_driver->suspend) {
1393                 ret = cpufreq_driver->suspend(cpu_policy);
1394                 if (ret)
1395                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1396                                         "step on CPU %u\n", cpu_policy->cpu);
1397         }
1398
1399         cpufreq_cpu_put(cpu_policy);
1400         return ret;
1401 }
1402
1403 /**
1404  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1405  *
1406  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1407  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1408  *          restored. It will verify that the current freq is in sync with
1409  *          what we believe it to be. This is a bit later than when it
1410  *          should be, but nonethteless it's better than calling
1411  *          cpufreq_driver->get() here which might re-enable interrupts...
1412  *
1413  * This function is only executed for the boot CPU.  The other CPUs have not
1414  * been turned on yet.
1415  */
1416 static void cpufreq_bp_resume(void)
1417 {
1418         int ret = 0;
1419
1420         int cpu = smp_processor_id();
1421         struct cpufreq_policy *cpu_policy;
1422
1423         pr_debug("resuming cpu %u\n", cpu);
1424
1425         /* If there's no policy for the boot CPU, we have nothing to do. */
1426         cpu_policy = cpufreq_cpu_get(cpu);
1427         if (!cpu_policy)
1428                 return;
1429
1430         if (cpufreq_driver->resume) {
1431                 ret = cpufreq_driver->resume(cpu_policy);
1432                 if (ret) {
1433                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1434                                         "step on CPU %u\n", cpu_policy->cpu);
1435                         goto fail;
1436                 }
1437         }
1438
1439         schedule_work(&cpu_policy->update);
1440
1441 fail:
1442         cpufreq_cpu_put(cpu_policy);
1443 }
1444
1445 static struct syscore_ops cpufreq_syscore_ops = {
1446         .suspend        = cpufreq_bp_suspend,
1447         .resume         = cpufreq_bp_resume,
1448 };
1449
1450 /**
1451  *      cpufreq_get_current_driver - return current driver's name
1452  *
1453  *      Return the name string of the currently loaded cpufreq driver
1454  *      or NULL, if none.
1455  */
1456 const char *cpufreq_get_current_driver(void)
1457 {
1458         if (cpufreq_driver)
1459                 return cpufreq_driver->name;
1460
1461         return NULL;
1462 }
1463 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1464
1465 /*********************************************************************
1466  *                     NOTIFIER LISTS INTERFACE                      *
1467  *********************************************************************/
1468
1469 /**
1470  *      cpufreq_register_notifier - register a driver with cpufreq
1471  *      @nb: notifier function to register
1472  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1473  *
1474  *      Add a driver to one of two lists: either a list of drivers that
1475  *      are notified about clock rate changes (once before and once after
1476  *      the transition), or a list of drivers that are notified about
1477  *      changes in cpufreq policy.
1478  *
1479  *      This function may sleep, and has the same return conditions as
1480  *      blocking_notifier_chain_register.
1481  */
1482 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1483 {
1484         int ret;
1485
1486         if (cpufreq_disabled())
1487                 return -EINVAL;
1488
1489         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1490
1491         switch (list) {
1492         case CPUFREQ_TRANSITION_NOTIFIER:
1493                 ret = srcu_notifier_chain_register(
1494                                 &cpufreq_transition_notifier_list, nb);
1495                 break;
1496         case CPUFREQ_POLICY_NOTIFIER:
1497                 ret = blocking_notifier_chain_register(
1498                                 &cpufreq_policy_notifier_list, nb);
1499                 break;
1500         default:
1501                 ret = -EINVAL;
1502         }
1503
1504         return ret;
1505 }
1506 EXPORT_SYMBOL(cpufreq_register_notifier);
1507
1508 /**
1509  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1510  *      @nb: notifier block to be unregistered
1511  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1512  *
1513  *      Remove a driver from the CPU frequency notifier list.
1514  *
1515  *      This function may sleep, and has the same return conditions as
1516  *      blocking_notifier_chain_unregister.
1517  */
1518 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1519 {
1520         int ret;
1521
1522         if (cpufreq_disabled())
1523                 return -EINVAL;
1524
1525         switch (list) {
1526         case CPUFREQ_TRANSITION_NOTIFIER:
1527                 ret = srcu_notifier_chain_unregister(
1528                                 &cpufreq_transition_notifier_list, nb);
1529                 break;
1530         case CPUFREQ_POLICY_NOTIFIER:
1531                 ret = blocking_notifier_chain_unregister(
1532                                 &cpufreq_policy_notifier_list, nb);
1533                 break;
1534         default:
1535                 ret = -EINVAL;
1536         }
1537
1538         return ret;
1539 }
1540 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1541
1542
1543 /*********************************************************************
1544  *                              GOVERNORS                            *
1545  *********************************************************************/
1546
1547 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1548                             unsigned int target_freq,
1549                             unsigned int relation)
1550 {
1551         int retval = -EINVAL;
1552         unsigned int old_target_freq = target_freq;
1553
1554         if (cpufreq_disabled())
1555                 return -ENODEV;
1556         if (policy->transition_ongoing)
1557                 return -EBUSY;
1558
1559         /* Make sure that target_freq is within supported range */
1560         if (target_freq > policy->max)
1561                 target_freq = policy->max;
1562         if (target_freq < policy->min)
1563                 target_freq = policy->min;
1564
1565         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1566                         policy->cpu, target_freq, relation, old_target_freq);
1567
1568         if (target_freq == policy->cur)
1569                 return 0;
1570
1571         if (cpufreq_driver->target)
1572                 retval = cpufreq_driver->target(policy, target_freq, relation);
1573
1574         return retval;
1575 }
1576 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1577
1578 int cpufreq_driver_target(struct cpufreq_policy *policy,
1579                           unsigned int target_freq,
1580                           unsigned int relation)
1581 {
1582         int ret = -EINVAL;
1583
1584         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1585                 goto fail;
1586
1587         ret = __cpufreq_driver_target(policy, target_freq, relation);
1588
1589         unlock_policy_rwsem_write(policy->cpu);
1590
1591 fail:
1592         return ret;
1593 }
1594 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1595
1596 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1597 {
1598         if (cpufreq_disabled())
1599                 return 0;
1600
1601         if (!cpufreq_driver->getavg)
1602                 return 0;
1603
1604         return cpufreq_driver->getavg(policy, cpu);
1605 }
1606 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1607
1608 /*
1609  * when "event" is CPUFREQ_GOV_LIMITS
1610  */
1611
1612 static int __cpufreq_governor(struct cpufreq_policy *policy,
1613                                         unsigned int event)
1614 {
1615         int ret;
1616
1617         /* Only must be defined when default governor is known to have latency
1618            restrictions, like e.g. conservative or ondemand.
1619            That this is the case is already ensured in Kconfig
1620         */
1621 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1622         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1623 #else
1624         struct cpufreq_governor *gov = NULL;
1625 #endif
1626
1627         if (policy->governor->max_transition_latency &&
1628             policy->cpuinfo.transition_latency >
1629             policy->governor->max_transition_latency) {
1630                 if (!gov)
1631                         return -EINVAL;
1632                 else {
1633                         printk(KERN_WARNING "%s governor failed, too long"
1634                                " transition latency of HW, fallback"
1635                                " to %s governor\n",
1636                                policy->governor->name,
1637                                gov->name);
1638                         policy->governor = gov;
1639                 }
1640         }
1641
1642         if (!try_module_get(policy->governor->owner))
1643                 return -EINVAL;
1644
1645         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1646                                                 policy->cpu, event);
1647
1648         mutex_lock(&cpufreq_governor_lock);
1649         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1650             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1651                 mutex_unlock(&cpufreq_governor_lock);
1652                 return -EBUSY;
1653         }
1654
1655         if (event == CPUFREQ_GOV_STOP)
1656                 policy->governor_enabled = false;
1657         else if (event == CPUFREQ_GOV_START)
1658                 policy->governor_enabled = true;
1659
1660         mutex_unlock(&cpufreq_governor_lock);
1661
1662         ret = policy->governor->governor(policy, event);
1663
1664         if (!ret) {
1665                 if (event == CPUFREQ_GOV_POLICY_INIT)
1666                         policy->governor->initialized++;
1667                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1668                         policy->governor->initialized--;
1669         } else {
1670                 /* Restore original values */
1671                 mutex_lock(&cpufreq_governor_lock);
1672                 if (event == CPUFREQ_GOV_STOP)
1673                         policy->governor_enabled = true;
1674                 else if (event == CPUFREQ_GOV_START)
1675                         policy->governor_enabled = false;
1676                 mutex_unlock(&cpufreq_governor_lock);
1677         }
1678
1679         /* we keep one module reference alive for
1680                         each CPU governed by this CPU */
1681         if ((event != CPUFREQ_GOV_START) || ret)
1682                 module_put(policy->governor->owner);
1683         if ((event == CPUFREQ_GOV_STOP) && !ret)
1684                 module_put(policy->governor->owner);
1685
1686         return ret;
1687 }
1688
1689 int cpufreq_register_governor(struct cpufreq_governor *governor)
1690 {
1691         int err;
1692
1693         if (!governor)
1694                 return -EINVAL;
1695
1696         if (cpufreq_disabled())
1697                 return -ENODEV;
1698
1699         mutex_lock(&cpufreq_governor_mutex);
1700
1701         governor->initialized = 0;
1702         err = -EBUSY;
1703         if (__find_governor(governor->name) == NULL) {
1704                 err = 0;
1705                 list_add(&governor->governor_list, &cpufreq_governor_list);
1706         }
1707
1708         mutex_unlock(&cpufreq_governor_mutex);
1709         return err;
1710 }
1711 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1712
1713 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1714 {
1715 #ifdef CONFIG_HOTPLUG_CPU
1716         int cpu;
1717 #endif
1718
1719         if (!governor)
1720                 return;
1721
1722         if (cpufreq_disabled())
1723                 return;
1724
1725 #ifdef CONFIG_HOTPLUG_CPU
1726         for_each_present_cpu(cpu) {
1727                 if (cpu_online(cpu))
1728                         continue;
1729                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1730                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1731         }
1732 #endif
1733
1734         mutex_lock(&cpufreq_governor_mutex);
1735         list_del(&governor->governor_list);
1736         mutex_unlock(&cpufreq_governor_mutex);
1737         return;
1738 }
1739 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1740
1741
1742 /*********************************************************************
1743  *                          POLICY INTERFACE                         *
1744  *********************************************************************/
1745
1746 /**
1747  * cpufreq_get_policy - get the current cpufreq_policy
1748  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1749  *      is written
1750  *
1751  * Reads the current cpufreq policy.
1752  */
1753 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1754 {
1755         struct cpufreq_policy *cpu_policy;
1756         if (!policy)
1757                 return -EINVAL;
1758
1759         cpu_policy = cpufreq_cpu_get(cpu);
1760         if (!cpu_policy)
1761                 return -EINVAL;
1762
1763         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1764
1765         cpufreq_cpu_put(cpu_policy);
1766         return 0;
1767 }
1768 EXPORT_SYMBOL(cpufreq_get_policy);
1769
1770 /*
1771  * data   : current policy.
1772  * policy : policy to be set.
1773  */
1774 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1775                                 struct cpufreq_policy *policy)
1776 {
1777         int ret = 0, failed = 1;
1778
1779         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1780                 policy->min, policy->max);
1781
1782         memcpy(&policy->cpuinfo, &data->cpuinfo,
1783                                 sizeof(struct cpufreq_cpuinfo));
1784
1785         if (policy->min > data->max || policy->max < data->min) {
1786                 ret = -EINVAL;
1787                 goto error_out;
1788         }
1789
1790         /* verify the cpu speed can be set within this limit */
1791         ret = cpufreq_driver->verify(policy);
1792         if (ret)
1793                 goto error_out;
1794
1795         /* adjust if necessary - all reasons */
1796         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1797                         CPUFREQ_ADJUST, policy);
1798
1799         /* adjust if necessary - hardware incompatibility*/
1800         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1801                         CPUFREQ_INCOMPATIBLE, policy);
1802
1803         /*
1804          * verify the cpu speed can be set within this limit, which might be
1805          * different to the first one
1806          */
1807         ret = cpufreq_driver->verify(policy);
1808         if (ret)
1809                 goto error_out;
1810
1811         /* notification of the new policy */
1812         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1813                         CPUFREQ_NOTIFY, policy);
1814
1815         data->min = policy->min;
1816         data->max = policy->max;
1817
1818         pr_debug("new min and max freqs are %u - %u kHz\n",
1819                                         data->min, data->max);
1820
1821         if (cpufreq_driver->setpolicy) {
1822                 data->policy = policy->policy;
1823                 pr_debug("setting range\n");
1824                 ret = cpufreq_driver->setpolicy(policy);
1825         } else {
1826                 if (policy->governor != data->governor) {
1827                         /* save old, working values */
1828                         struct cpufreq_governor *old_gov = data->governor;
1829
1830                         pr_debug("governor switch\n");
1831
1832                         /* end old governor */
1833                         if (data->governor) {
1834                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1835                                 unlock_policy_rwsem_write(policy->cpu);
1836                                 __cpufreq_governor(data,
1837                                                 CPUFREQ_GOV_POLICY_EXIT);
1838                                 lock_policy_rwsem_write(policy->cpu);
1839                         }
1840
1841                         /* start new governor */
1842                         data->governor = policy->governor;
1843                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1844                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1845                                         failed = 0;
1846                                 } else {
1847                                         unlock_policy_rwsem_write(policy->cpu);
1848                                         __cpufreq_governor(data,
1849                                                         CPUFREQ_GOV_POLICY_EXIT);
1850                                         lock_policy_rwsem_write(policy->cpu);
1851                                 }
1852                         }
1853
1854                         if (failed) {
1855                                 /* new governor failed, so re-start old one */
1856                                 pr_debug("starting governor %s failed\n",
1857                                                         data->governor->name);
1858                                 if (old_gov) {
1859                                         data->governor = old_gov;
1860                                         __cpufreq_governor(data,
1861                                                         CPUFREQ_GOV_POLICY_INIT);
1862                                         __cpufreq_governor(data,
1863                                                            CPUFREQ_GOV_START);
1864                                 }
1865                                 ret = -EINVAL;
1866                                 goto error_out;
1867                         }
1868                         /* might be a policy change, too, so fall through */
1869                 }
1870                 pr_debug("governor: change or update limits\n");
1871                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1872         }
1873
1874 error_out:
1875         return ret;
1876 }
1877
1878 /**
1879  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1880  *      @cpu: CPU which shall be re-evaluated
1881  *
1882  *      Useful for policy notifiers which have different necessities
1883  *      at different times.
1884  */
1885 int cpufreq_update_policy(unsigned int cpu)
1886 {
1887         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1888         struct cpufreq_policy policy;
1889         int ret;
1890
1891         if (!data) {
1892                 ret = -ENODEV;
1893                 goto no_policy;
1894         }
1895
1896         if (unlikely(lock_policy_rwsem_write(cpu))) {
1897                 ret = -EINVAL;
1898                 goto fail;
1899         }
1900
1901         pr_debug("updating policy for CPU %u\n", cpu);
1902         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1903         policy.min = data->user_policy.min;
1904         policy.max = data->user_policy.max;
1905         policy.policy = data->user_policy.policy;
1906         policy.governor = data->user_policy.governor;
1907
1908         /*
1909          * BIOS might change freq behind our back
1910          * -> ask driver for current freq and notify governors about a change
1911          */
1912         if (cpufreq_driver->get) {
1913                 policy.cur = cpufreq_driver->get(cpu);
1914                 if (!data->cur) {
1915                         pr_debug("Driver did not initialize current freq");
1916                         data->cur = policy.cur;
1917                 } else {
1918                         if (data->cur != policy.cur && cpufreq_driver->target)
1919                                 cpufreq_out_of_sync(cpu, data->cur,
1920                                                                 policy.cur);
1921                 }
1922         }
1923
1924         ret = __cpufreq_set_policy(data, &policy);
1925
1926         unlock_policy_rwsem_write(cpu);
1927
1928 fail:
1929         cpufreq_cpu_put(data);
1930 no_policy:
1931         return ret;
1932 }
1933 EXPORT_SYMBOL(cpufreq_update_policy);
1934
1935 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1936                                         unsigned long action, void *hcpu)
1937 {
1938         unsigned int cpu = (unsigned long)hcpu;
1939         struct device *dev;
1940
1941         dev = get_cpu_device(cpu);
1942         if (dev) {
1943                 switch (action) {
1944                 case CPU_ONLINE:
1945                         cpufreq_add_dev(dev, NULL);
1946                         break;
1947                 case CPU_DOWN_PREPARE:
1948                 case CPU_UP_CANCELED_FROZEN:
1949                         __cpufreq_remove_dev(dev, NULL);
1950                         break;
1951                 case CPU_DOWN_FAILED:
1952                         cpufreq_add_dev(dev, NULL);
1953                         break;
1954                 }
1955         }
1956         return NOTIFY_OK;
1957 }
1958
1959 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1960         .notifier_call = cpufreq_cpu_callback,
1961 };
1962
1963 /*********************************************************************
1964  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1965  *********************************************************************/
1966
1967 /**
1968  * cpufreq_register_driver - register a CPU Frequency driver
1969  * @driver_data: A struct cpufreq_driver containing the values#
1970  * submitted by the CPU Frequency driver.
1971  *
1972  * Registers a CPU Frequency driver to this core code. This code
1973  * returns zero on success, -EBUSY when another driver got here first
1974  * (and isn't unregistered in the meantime).
1975  *
1976  */
1977 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1978 {
1979         unsigned long flags;
1980         int ret;
1981
1982         if (cpufreq_disabled())
1983                 return -ENODEV;
1984
1985         if (!driver_data || !driver_data->verify || !driver_data->init ||
1986             ((!driver_data->setpolicy) && (!driver_data->target)))
1987                 return -EINVAL;
1988
1989         pr_debug("trying to register driver %s\n", driver_data->name);
1990
1991         if (driver_data->setpolicy)
1992                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1993
1994         write_lock_irqsave(&cpufreq_driver_lock, flags);
1995         if (cpufreq_driver) {
1996                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1997                 return -EBUSY;
1998         }
1999         cpufreq_driver = driver_data;
2000         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2001
2002         ret = subsys_interface_register(&cpufreq_interface);
2003         if (ret)
2004                 goto err_null_driver;
2005
2006         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2007                 int i;
2008                 ret = -ENODEV;
2009
2010                 /* check for at least one working CPU */
2011                 for (i = 0; i < nr_cpu_ids; i++)
2012                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2013                                 ret = 0;
2014                                 break;
2015                         }
2016
2017                 /* if all ->init() calls failed, unregister */
2018                 if (ret) {
2019                         pr_debug("no CPU initialized for driver %s\n",
2020                                                         driver_data->name);
2021                         goto err_if_unreg;
2022                 }
2023         }
2024
2025         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2026         pr_debug("driver %s up and running\n", driver_data->name);
2027
2028         return 0;
2029 err_if_unreg:
2030         subsys_interface_unregister(&cpufreq_interface);
2031 err_null_driver:
2032         write_lock_irqsave(&cpufreq_driver_lock, flags);
2033         cpufreq_driver = NULL;
2034         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2035         return ret;
2036 }
2037 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2038
2039 /**
2040  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2041  *
2042  * Unregister the current CPUFreq driver. Only call this if you have
2043  * the right to do so, i.e. if you have succeeded in initialising before!
2044  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2045  * currently not initialised.
2046  */
2047 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2048 {
2049         unsigned long flags;
2050
2051         if (!cpufreq_driver || (driver != cpufreq_driver))
2052                 return -EINVAL;
2053
2054         pr_debug("unregistering driver %s\n", driver->name);
2055
2056         subsys_interface_unregister(&cpufreq_interface);
2057         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2058
2059         write_lock_irqsave(&cpufreq_driver_lock, flags);
2060         cpufreq_driver = NULL;
2061         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2062
2063         return 0;
2064 }
2065 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2066
2067 static int __init cpufreq_core_init(void)
2068 {
2069         int cpu;
2070
2071         if (cpufreq_disabled())
2072                 return -ENODEV;
2073
2074         for_each_possible_cpu(cpu) {
2075                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2076                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2077         }
2078
2079         cpufreq_global_kobject = kobject_create();
2080         BUG_ON(!cpufreq_global_kobject);
2081         register_syscore_ops(&cpufreq_syscore_ops);
2082
2083         return 0;
2084 }
2085 core_initcall(cpufreq_core_init);