Pileus Git - ~andy/linux/blob - drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <asm/cputime.h>
  21 #include <linux/kernel.h>
  22 #include <linux/kernel_stat.h>
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/notifier.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/delay.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/tick.h>
  31 #include <linux/device.h>
  32 #include <linux/slab.h>
  33 #include <linux/cpu.h>
  34 #include <linux/completion.h>
  35 #include <linux/mutex.h>
  36 #include <linux/syscore_ops.h>
  37
  38 #include <trace/events/power.h>
  39
  40 /**
  41  * The "cpufreq driver" - the arch- or hardware-dependent low
  42  * level driver of CPUFreq support, and its spinlock. This lock
  43  * also protects the cpufreq_cpu_data array.
  44  */
  45 static struct cpufreq_driver *cpufreq_driver;
  46 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  47 static DEFINE_RWLOCK(cpufreq_driver_lock);
  48 static DEFINE_MUTEX(cpufreq_governor_lock);
  49
  50 #ifdef CONFIG_HOTPLUG_CPU
  51 /* This one keeps track of the previously set governor of a removed CPU */
  52 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  53 #endif
  54
  55 /*
  56  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  57  * all cpufreq/hotplug/workqueue/etc related lock issues.
  58  *
  59  * The rules for this semaphore:
  60  * - Any routine that wants to read from the policy structure will
  61  *   do a down_read on this semaphore.
  62  * - Any routine that will write to the policy structure and/or may take away
  63  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  64  *   mode before doing so.
  65  *
  66  * Additional rules:
  67  * - Governor routines that can be called in cpufreq hotplug path should not
  68  *   take this sem as top level hotplug notifier handler takes this.
  69  * - Lock should not be held across
  70  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  71  */
  72 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  73 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  74
  75 #define lock_policy_rwsem(mode, cpu)                                    \
  76 static int lock_policy_rwsem_##mode(int cpu)                            \
  77 {                                                                       \
  78         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  79         BUG_ON(policy_cpu == -1);                                       \
  80         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86 lock_policy_rwsem(write, cpu);
  87
  88 #define unlock_policy_rwsem(mode, cpu)                                  \
  89 static void unlock_policy_rwsem_##mode(int cpu)                         \
  90 {                                                                       \
  91         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  92         BUG_ON(policy_cpu == -1);                                       \
  93         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
  94 }
  95
  96 unlock_policy_rwsem(read, cpu);
  97 unlock_policy_rwsem(write, cpu);
  98
  99 /* internal prototypes */
 100 static int __cpufreq_governor(struct cpufreq_policy *policy,
 101                 unsigned int event);
 102 static unsigned int __cpufreq_get(unsigned int cpu);
 103 static void handle_update(struct work_struct *work);
 104
 105 /**
 106  * Two notifier lists: the "policy" list is involved in the
 107  * validation process for a new CPU frequency policy; the
 108  * "transition" list for kernel code that needs to handle
 109  * changes to devices when the CPU clock speed changes.
 110  * The mutex locks both lists.
 111  */
 112 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 113 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 114
 115 static bool init_cpufreq_transition_notifier_list_called;
 116 static int __init init_cpufreq_transition_notifier_list(void)
 117 {
 118         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 119         init_cpufreq_transition_notifier_list_called = true;
 120         return 0;
 121 }
 122 pure_initcall(init_cpufreq_transition_notifier_list);
 123
 124 static int off __read_mostly;
 125 static int cpufreq_disabled(void)
 126 {
 127         return off;
 128 }
 129 void disable_cpufreq(void)
 130 {
 131         off = 1;
 132 }
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 bool have_governor_per_policy(void)
 137 {
 138         return cpufreq_driver->have_governor_per_policy;
 139 }
 140 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 141
 142 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 143 {
 144         if (have_governor_per_policy())
 145                 return &policy->kobj;
 146         else
 147                 return cpufreq_global_kobject;
 148 }
 149 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 150
 151 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 152 {
 153         u64 idle_time;
 154         u64 cur_wall_time;
 155         u64 busy_time;
 156
 157         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 158
 159         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 165
 166         idle_time = cur_wall_time - busy_time;
 167         if (wall)
 168                 *wall = cputime_to_usecs(cur_wall_time);
 169
 170         return cputime_to_usecs(idle_time);
 171 }
 172
 173 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 174 {
 175         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 176
 177         if (idle_time == -1ULL)
 178                 return get_cpu_idle_time_jiffy(cpu, wall);
 179         else if (!io_busy)
 180                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 181
 182         return idle_time;
 183 }
 184 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 185
 186 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 187 {
 188         struct cpufreq_policy *data;
 189         unsigned long flags;
 190
 191         if (cpu >= nr_cpu_ids)
 192                 goto err_out;
 193
 194         /* get the cpufreq driver */
 195         read_lock_irqsave(&cpufreq_driver_lock, flags);
 196
 197         if (!cpufreq_driver)
 198                 goto err_out_unlock;
 199
 200         if (!try_module_get(cpufreq_driver->owner))
 201                 goto err_out_unlock;
 202
 203         /* get the CPU */
 204         data = per_cpu(cpufreq_cpu_data, cpu);
 205
 206         if (!data)
 207                 goto err_out_put_module;
 208
 209         if (!sysfs && !kobject_get(&data->kobj))
 210                 goto err_out_put_module;
 211
 212         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 213         return data;
 214
 215 err_out_put_module:
 216         module_put(cpufreq_driver->owner);
 217 err_out_unlock:
 218         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 219 err_out:
 220         return NULL;
 221 }
 222
 223 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 224 {
 225         if (cpufreq_disabled())
 226                 return NULL;
 227
 228         return __cpufreq_cpu_get(cpu, false);
 229 }
 230 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 231
 232 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 233 {
 234         return __cpufreq_cpu_get(cpu, true);
 235 }
 236
 237 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 238 {
 239         if (!sysfs)
 240                 kobject_put(&data->kobj);
 241         module_put(cpufreq_driver->owner);
 242 }
 243
 244 void cpufreq_cpu_put(struct cpufreq_policy *data)
 245 {
 246         if (cpufreq_disabled())
 247                 return;
 248
 249         __cpufreq_cpu_put(data, false);
 250 }
 251 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 252
 253 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 254 {
 255         __cpufreq_cpu_put(data, true);
 256 }
 257
 258 /*********************************************************************
 259  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 260  *********************************************************************/
 261
 262 /**
 263  * adjust_jiffies - adjust the system "loops_per_jiffy"
 264  *
 265  * This function alters the system "loops_per_jiffy" for the clock
 266  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 267  * systems as each CPU might be scaled differently. So, use the arch
 268  * per-CPU loops_per_jiffy value wherever possible.
 269  */
 270 #ifndef CONFIG_SMP
 271 static unsigned long l_p_j_ref;
 272 static unsigned int l_p_j_ref_freq;
 273
 274 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 275 {
 276         if (ci->flags & CPUFREQ_CONST_LOOPS)
 277                 return;
 278
 279         if (!l_p_j_ref_freq) {
 280                 l_p_j_ref = loops_per_jiffy;
 281                 l_p_j_ref_freq = ci->old;
 282                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 283                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 284         }
 285         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 286             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 287                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 288                                                                 ci->new);
 289                 pr_debug("scaling loops_per_jiffy to %lu "
 290                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 291         }
 292 }
 293 #else
 294 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 295 {
 296         return;
 297 }
 298 #endif
 299
 300 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 301                 struct cpufreq_freqs *freqs, unsigned int state)
 302 {
 303         BUG_ON(irqs_disabled());
 304
 305         if (cpufreq_disabled())
 306                 return;
 307
 308         freqs->flags = cpufreq_driver->flags;
 309         pr_debug("notification %u of frequency transition to %u kHz\n",
 310                 state, freqs->new);
 311
 312         switch (state) {
 313
 314         case CPUFREQ_PRECHANGE:
 315                 if (WARN(policy->transition_ongoing,
 316                                 "In middle of another frequency transition\n"))
 317                         return;
 318
 319                 policy->transition_ongoing = true;
 320
 321                 /* detect if the driver reported a value as "old frequency"
 322                  * which is not equal to what the cpufreq core thinks is
 323                  * "old frequency".
 324                  */
 325                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 326                         if ((policy) && (policy->cpu == freqs->cpu) &&
 327                             (policy->cur) && (policy->cur != freqs->old)) {
 328                                 pr_debug("Warning: CPU frequency is"
 329                                         " %u, cpufreq assumed %u kHz.\n",
 330                                         freqs->old, policy->cur);
 331                                 freqs->old = policy->cur;
 332                         }
 333                 }
 334                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 335                                 CPUFREQ_PRECHANGE, freqs);
 336                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 337                 break;
 338
 339         case CPUFREQ_POSTCHANGE:
 340                 if (WARN(!policy->transition_ongoing,
 341                                 "No frequency transition in progress\n"))
 342                         return;
 343
 344                 policy->transition_ongoing = false;
 345
 346                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 347                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 348                         (unsigned long)freqs->cpu);
 349                 trace_cpu_frequency(freqs->new, freqs->cpu);
 350                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 351                                 CPUFREQ_POSTCHANGE, freqs);
 352                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 353                         policy->cur = freqs->new;
 354                 break;
 355         }
 356 }
 357
 358 /**
 359  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 360  * on frequency transition.
 361  *
 362  * This function calls the transition notifiers and the "adjust_jiffies"
 363  * function. It is called twice on all CPU frequency changes that have
 364  * external effects.
 365  */
 366 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 367                 struct cpufreq_freqs *freqs, unsigned int state)
 368 {
 369         for_each_cpu(freqs->cpu, policy->cpus)
 370                 __cpufreq_notify_transition(policy, freqs, state);
 371 }
 372 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 373
 374
 375 /*********************************************************************
 376  *                          SYSFS INTERFACE                          *
 377  *********************************************************************/
 378
 379 static struct cpufreq_governor *__find_governor(const char *str_governor)
 380 {
 381         struct cpufreq_governor *t;
 382
 383         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 384                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 385                         return t;
 386
 387         return NULL;
 388 }
 389
 390 /**
 391  * cpufreq_parse_governor - parse a governor string
 392  */
 393 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 394                                 struct cpufreq_governor **governor)
 395 {
 396         int err = -EINVAL;
 397
 398         if (!cpufreq_driver)
 399                 goto out;
 400
 401         if (cpufreq_driver->setpolicy) {
 402                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 403                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 404                         err = 0;
 405                 } else if (!strnicmp(str_governor, "powersave",
 406                                                 CPUFREQ_NAME_LEN)) {
 407                         *policy = CPUFREQ_POLICY_POWERSAVE;
 408                         err = 0;
 409                 }
 410         } else if (cpufreq_driver->target) {
 411                 struct cpufreq_governor *t;
 412
 413                 mutex_lock(&cpufreq_governor_mutex);
 414
 415                 t = __find_governor(str_governor);
 416
 417                 if (t == NULL) {
 418                         int ret;
 419
 420                         mutex_unlock(&cpufreq_governor_mutex);
 421                         ret = request_module("cpufreq_%s", str_governor);
 422                         mutex_lock(&cpufreq_governor_mutex);
 423
 424                         if (ret == 0)
 425                                 t = __find_governor(str_governor);
 426                 }
 427
 428                 if (t != NULL) {
 429                         *governor = t;
 430                         err = 0;
 431                 }
 432
 433                 mutex_unlock(&cpufreq_governor_mutex);
 434         }
 435 out:
 436         return err;
 437 }
 438
 439 /**
 440  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 441  * print out cpufreq information
 442  *
 443  * Write out information from cpufreq_driver->policy[cpu]; object must be
 444  * "unsigned int".
 445  */
 446
 447 #define show_one(file_name, object)                     \
 448 static ssize_t show_##file_name                         \
 449 (struct cpufreq_policy *policy, char *buf)              \
 450 {                                                       \
 451         return sprintf(buf, "%u\n", policy->object);    \
 452 }
 453
 454 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 455 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 456 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 457 show_one(scaling_min_freq, min);
 458 show_one(scaling_max_freq, max);
 459 show_one(scaling_cur_freq, cur);
 460
 461 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 462                                 struct cpufreq_policy *policy);
 463
 464 /**
 465  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 466  */
 467 #define store_one(file_name, object)                    \
 468 static ssize_t store_##file_name                                        \
 469 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 470 {                                                                       \
 471         unsigned int ret;                                               \
 472         struct cpufreq_policy new_policy;                               \
 473                                                                         \
 474         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 475         if (ret)                                                        \
 476                 return -EINVAL;                                         \
 477                                                                         \
 478         ret = sscanf(buf, "%u", &new_policy.object);                    \
 479         if (ret != 1)                                                   \
 480                 return -EINVAL;                                         \
 481                                                                         \
 482         ret = __cpufreq_set_policy(policy, &new_policy);                \
 483         policy->user_policy.object = policy->object;                    \
 484                                                                         \
 485         return ret ? ret : count;                                       \
 486 }
 487
 488 store_one(scaling_min_freq, min);
 489 store_one(scaling_max_freq, max);
 490
 491 /**
 492  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 493  */
 494 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 495                                         char *buf)
 496 {
 497         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 498         if (!cur_freq)
 499                 return sprintf(buf, "<unknown>");
 500         return sprintf(buf, "%u\n", cur_freq);
 501 }
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 507 {
 508         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 509                 return sprintf(buf, "powersave\n");
 510         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 511                 return sprintf(buf, "performance\n");
 512         else if (policy->governor)
 513                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 514                                 policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518 /**
 519  * store_scaling_governor - store policy for the specified CPU
 520  */
 521 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 522                                         const char *buf, size_t count)
 523 {
 524         unsigned int ret;
 525         char    str_governor[16];
 526         struct cpufreq_policy new_policy;
 527
 528         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 529         if (ret)
 530                 return ret;
 531
 532         ret = sscanf(buf, "%15s", str_governor);
 533         if (ret != 1)
 534                 return -EINVAL;
 535
 536         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 537                                                 &new_policy.governor))
 538                 return -EINVAL;
 539
 540         /*
 541          * Do not use cpufreq_set_policy here or the user_policy.max
 542          * will be wrongly overridden
 543          */
 544         ret = __cpufreq_set_policy(policy, &new_policy);
 545
 546         policy->user_policy.policy = policy->policy;
 547         policy->user_policy.governor = policy->governor;
 548
 549         if (ret)
 550                 return ret;
 551         else
 552                 return count;
 553 }
 554
 555 /**
 556  * show_scaling_driver - show the cpufreq driver currently loaded
 557  */
 558 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 559 {
 560         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 561 }
 562
 563 /**
 564  * show_scaling_available_governors - show the available CPUfreq governors
 565  */
 566 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 567                                                 char *buf)
 568 {
 569         ssize_t i = 0;
 570         struct cpufreq_governor *t;
 571
 572         if (!cpufreq_driver->target) {
 573                 i += sprintf(buf, "performance powersave");
 574                 goto out;
 575         }
 576
 577         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 578                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 579                     - (CPUFREQ_NAME_LEN + 2)))
 580                         goto out;
 581                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 582         }
 583 out:
 584         i += sprintf(&buf[i], "\n");
 585         return i;
 586 }
 587
 588 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 589 {
 590         ssize_t i = 0;
 591         unsigned int cpu;
 592
 593         for_each_cpu(cpu, mask) {
 594                 if (i)
 595                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 596                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 597                 if (i >= (PAGE_SIZE - 5))
 598                         break;
 599         }
 600         i += sprintf(&buf[i], "\n");
 601         return i;
 602 }
 603 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 604
 605 /**
 606  * show_related_cpus - show the CPUs affected by each transition even if
 607  * hw coordination is in use
 608  */
 609 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 610 {
 611         return cpufreq_show_cpus(policy->related_cpus, buf);
 612 }
 613
 614 /**
 615  * show_affected_cpus - show the CPUs affected by each transition
 616  */
 617 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 618 {
 619         return cpufreq_show_cpus(policy->cpus, buf);
 620 }
 621
 622 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 623                                         const char *buf, size_t count)
 624 {
 625         unsigned int freq = 0;
 626         unsigned int ret;
 627
 628         if (!policy->governor || !policy->governor->store_setspeed)
 629                 return -EINVAL;
 630
 631         ret = sscanf(buf, "%u", &freq);
 632         if (ret != 1)
 633                 return -EINVAL;
 634
 635         policy->governor->store_setspeed(policy, freq);
 636
 637         return count;
 638 }
 639
 640 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 641 {
 642         if (!policy->governor || !policy->governor->show_setspeed)
 643                 return sprintf(buf, "<unsupported>\n");
 644
 645         return policy->governor->show_setspeed(policy, buf);
 646 }
 647
 648 /**
 649  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 650  */
 651 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 652 {
 653         unsigned int limit;
 654         int ret;
 655         if (cpufreq_driver->bios_limit) {
 656                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 657                 if (!ret)
 658                         return sprintf(buf, "%u\n", limit);
 659         }
 660         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 661 }
 662
 663 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 664 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 665 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 666 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 667 cpufreq_freq_attr_ro(scaling_available_governors);
 668 cpufreq_freq_attr_ro(scaling_driver);
 669 cpufreq_freq_attr_ro(scaling_cur_freq);
 670 cpufreq_freq_attr_ro(bios_limit);
 671 cpufreq_freq_attr_ro(related_cpus);
 672 cpufreq_freq_attr_ro(affected_cpus);
 673 cpufreq_freq_attr_rw(scaling_min_freq);
 674 cpufreq_freq_attr_rw(scaling_max_freq);
 675 cpufreq_freq_attr_rw(scaling_governor);
 676 cpufreq_freq_attr_rw(scaling_setspeed);
 677
 678 static struct attribute *default_attrs[] = {
 679         &cpuinfo_min_freq.attr,
 680         &cpuinfo_max_freq.attr,
 681         &cpuinfo_transition_latency.attr,
 682         &scaling_min_freq.attr,
 683         &scaling_max_freq.attr,
 684         &affected_cpus.attr,
 685         &related_cpus.attr,
 686         &scaling_governor.attr,
 687         &scaling_driver.attr,
 688         &scaling_available_governors.attr,
 689         &scaling_setspeed.attr,
 690         NULL
 691 };
 692
 693 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 694 #define to_attr(a) container_of(a, struct freq_attr, attr)
 695
 696 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 697 {
 698         struct cpufreq_policy *policy = to_policy(kobj);
 699         struct freq_attr *fattr = to_attr(attr);
 700         ssize_t ret = -EINVAL;
 701         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 702         if (!policy)
 703                 goto no_policy;
 704
 705         if (lock_policy_rwsem_read(policy->cpu) < 0)
 706                 goto fail;
 707
 708         if (fattr->show)
 709                 ret = fattr->show(policy, buf);
 710         else
 711                 ret = -EIO;
 712
 713         unlock_policy_rwsem_read(policy->cpu);
 714 fail:
 715         cpufreq_cpu_put_sysfs(policy);
 716 no_policy:
 717         return ret;
 718 }
 719
 720 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 721                      const char *buf, size_t count)
 722 {
 723         struct cpufreq_policy *policy = to_policy(kobj);
 724         struct freq_attr *fattr = to_attr(attr);
 725         ssize_t ret = -EINVAL;
 726         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 727         if (!policy)
 728                 goto no_policy;
 729
 730         if (lock_policy_rwsem_write(policy->cpu) < 0)
 731                 goto fail;
 732
 733         if (fattr->store)
 734                 ret = fattr->store(policy, buf, count);
 735         else
 736                 ret = -EIO;
 737
 738         unlock_policy_rwsem_write(policy->cpu);
 739 fail:
 740         cpufreq_cpu_put_sysfs(policy);
 741 no_policy:
 742         return ret;
 743 }
 744
 745 static void cpufreq_sysfs_release(struct kobject *kobj)
 746 {
 747         struct cpufreq_policy *policy = to_policy(kobj);
 748         pr_debug("last reference is dropped\n");
 749         complete(&policy->kobj_unregister);
 750 }
 751
 752 static const struct sysfs_ops sysfs_ops = {
 753         .show   = show,
 754         .store  = store,
 755 };
 756
 757 static struct kobj_type ktype_cpufreq = {
 758         .sysfs_ops      = &sysfs_ops,
 759         .default_attrs  = default_attrs,
 760         .release        = cpufreq_sysfs_release,
 761 };
 762
 763 struct kobject *cpufreq_global_kobject;
 764 EXPORT_SYMBOL(cpufreq_global_kobject);
 765
 766 static int cpufreq_global_kobject_usage;
 767
 768 int cpufreq_get_global_kobject(void)
 769 {
 770         if (!cpufreq_global_kobject_usage++)
 771                 return kobject_add(cpufreq_global_kobject,
 772                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 773
 774         return 0;
 775 }
 776 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 777
 778 void cpufreq_put_global_kobject(void)
 779 {
 780         if (!--cpufreq_global_kobject_usage)
 781                 kobject_del(cpufreq_global_kobject);
 782 }
 783 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 784
 785 int cpufreq_sysfs_create_file(const struct attribute *attr)
 786 {
 787         int ret = cpufreq_get_global_kobject();
 788
 789         if (!ret) {
 790                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 791                 if (ret)
 792                         cpufreq_put_global_kobject();
 793         }
 794
 795         return ret;
 796 }
 797 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 798
 799 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 800 {
 801         sysfs_remove_file(cpufreq_global_kobject, attr);
 802         cpufreq_put_global_kobject();
 803 }
 804 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 805
 806 /* symlink affected CPUs */
 807 static int cpufreq_add_dev_symlink(unsigned int cpu,
 808                                    struct cpufreq_policy *policy)
 809 {
 810         unsigned int j;
 811         int ret = 0;
 812
 813         for_each_cpu(j, policy->cpus) {
 814                 struct cpufreq_policy *managed_policy;
 815                 struct device *cpu_dev;
 816
 817                 if (j == cpu)
 818                         continue;
 819
 820                 pr_debug("CPU %u already managed, adding link\n", j);
 821                 managed_policy = cpufreq_cpu_get(cpu);
 822                 cpu_dev = get_cpu_device(j);
 823                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 824                                         "cpufreq");
 825                 if (ret) {
 826                         cpufreq_cpu_put(managed_policy);
 827                         return ret;
 828                 }
 829         }
 830         return ret;
 831 }
 832
 833 static int cpufreq_add_dev_interface(unsigned int cpu,
 834                                      struct cpufreq_policy *policy,
 835                                      struct device *dev)
 836 {
 837         struct cpufreq_policy new_policy;
 838         struct freq_attr **drv_attr;
 839         unsigned long flags;
 840         int ret = 0;
 841         unsigned int j;
 842
 843         /* prepare interface data */
 844         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 845                                    &dev->kobj, "cpufreq");
 846         if (ret)
 847                 return ret;
 848
 849         /* set up files for this cpu device */
 850         drv_attr = cpufreq_driver->attr;
 851         while ((drv_attr) && (*drv_attr)) {
 852                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 853                 if (ret)
 854                         goto err_out_kobj_put;
 855                 drv_attr++;
 856         }
 857         if (cpufreq_driver->get) {
 858                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 859                 if (ret)
 860                         goto err_out_kobj_put;
 861         }
 862         if (cpufreq_driver->target) {
 863                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 864                 if (ret)
 865                         goto err_out_kobj_put;
 866         }
 867         if (cpufreq_driver->bios_limit) {
 868                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 869                 if (ret)
 870                         goto err_out_kobj_put;
 871         }
 872
 873         write_lock_irqsave(&cpufreq_driver_lock, flags);
 874         for_each_cpu(j, policy->cpus) {
 875                 per_cpu(cpufreq_cpu_data, j) = policy;
 876                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 877         }
 878         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 879
 880         ret = cpufreq_add_dev_symlink(cpu, policy);
 881         if (ret)
 882                 goto err_out_kobj_put;
 883
 884         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 885         /* assure that the starting sequence is run in __cpufreq_set_policy */
 886         policy->governor = NULL;
 887
 888         /* set default policy */
 889         ret = __cpufreq_set_policy(policy, &new_policy);
 890         policy->user_policy.policy = policy->policy;
 891         policy->user_policy.governor = policy->governor;
 892
 893         if (ret) {
 894                 pr_debug("setting policy failed\n");
 895                 if (cpufreq_driver->exit)
 896                         cpufreq_driver->exit(policy);
 897         }
 898         return ret;
 899
 900 err_out_kobj_put:
 901         kobject_put(&policy->kobj);
 902         wait_for_completion(&policy->kobj_unregister);
 903         return ret;
 904 }
 905
 906 #ifdef CONFIG_HOTPLUG_CPU
 907 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 908                                   struct device *dev)
 909 {
 910         struct cpufreq_policy *policy;
 911         int ret = 0, has_target = !!cpufreq_driver->target;
 912         unsigned long flags;
 913
 914         policy = cpufreq_cpu_get(sibling);
 915         WARN_ON(!policy);
 916
 917         if (has_target)
 918                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 919
 920         lock_policy_rwsem_write(sibling);
 921
 922         write_lock_irqsave(&cpufreq_driver_lock, flags);
 923
 924         cpumask_set_cpu(cpu, policy->cpus);
 925         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 926         per_cpu(cpufreq_cpu_data, cpu) = policy;
 927         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 928
 929         unlock_policy_rwsem_write(sibling);
 930
 931         if (has_target) {
 932                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 933                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 934         }
 935
 936         ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 937         if (ret) {
 938                 cpufreq_cpu_put(policy);
 939                 return ret;
 940         }
 941
 942         return 0;
 943 }
 944 #endif
 945
 946 /**
 947  * cpufreq_add_dev - add a CPU device
 948  *
 949  * Adds the cpufreq interface for a CPU device.
 950  *
 951  * The Oracle says: try running cpufreq registration/unregistration concurrently
 952  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 953  * mess up, but more thorough testing is needed. - Mathieu
 954  */
 955 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 956 {
 957         unsigned int j, cpu = dev->id;
 958         int ret = -ENOMEM;
 959         struct cpufreq_policy *policy;
 960         unsigned long flags;
 961 #ifdef CONFIG_HOTPLUG_CPU
 962         struct cpufreq_governor *gov;
 963         int sibling;
 964 #endif
 965
 966         if (cpu_is_offline(cpu))
 967                 return 0;
 968
 969         pr_debug("adding CPU %u\n", cpu);
 970
 971 #ifdef CONFIG_SMP
 972         /* check whether a different CPU already registered this
 973          * CPU because it is in the same boat. */
 974         policy = cpufreq_cpu_get(cpu);
 975         if (unlikely(policy)) {
 976                 cpufreq_cpu_put(policy);
 977                 return 0;
 978         }
 979
 980 #ifdef CONFIG_HOTPLUG_CPU
 981         /* Check if this cpu was hot-unplugged earlier and has siblings */
 982         read_lock_irqsave(&cpufreq_driver_lock, flags);
 983         for_each_online_cpu(sibling) {
 984                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 985                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
 986                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 987                         return cpufreq_add_policy_cpu(cpu, sibling, dev);
 988                 }
 989         }
 990         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 991 #endif
 992 #endif
 993
 994         if (!try_module_get(cpufreq_driver->owner)) {
 995                 ret = -EINVAL;
 996                 goto module_out;
 997         }
 998
 999         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
1000         if (!policy)
1001                 goto nomem_out;
1002
1003         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1004                 goto err_free_policy;
1005
1006         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1007                 goto err_free_cpumask;
1008
1009         policy->cpu = cpu;
1010         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1011         cpumask_copy(policy->cpus, cpumask_of(cpu));
1012
1013         /* Initially set CPU itself as the policy_cpu */
1014         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
1015
1016         init_completion(&policy->kobj_unregister);
1017         INIT_WORK(&policy->update, handle_update);
1018
1019         /* call driver. From then on the cpufreq must be able
1020          * to accept all calls to ->verify and ->setpolicy for this CPU
1021          */
1022         ret = cpufreq_driver->init(policy);
1023         if (ret) {
1024                 pr_debug("initialization failed\n");
1025                 goto err_set_policy_cpu;
1026         }
1027
1028         /* related cpus should atleast have policy->cpus */
1029         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1030
1031         /*
1032          * affected cpus must always be the one, which are online. We aren't
1033          * managing offline cpus here.
1034          */
1035         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1036
1037         policy->user_policy.min = policy->min;
1038         policy->user_policy.max = policy->max;
1039
1040         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1041                                      CPUFREQ_START, policy);
1042
1043 #ifdef CONFIG_HOTPLUG_CPU
1044         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1045         if (gov) {
1046                 policy->governor = gov;
1047                 pr_debug("Restoring governor %s for cpu %d\n",
1048                        policy->governor->name, cpu);
1049         }
1050 #endif
1051
1052         ret = cpufreq_add_dev_interface(cpu, policy, dev);
1053         if (ret)
1054                 goto err_out_unregister;
1055
1056         kobject_uevent(&policy->kobj, KOBJ_ADD);
1057         module_put(cpufreq_driver->owner);
1058         pr_debug("initialization complete\n");
1059
1060         return 0;
1061
1062 err_out_unregister:
1063         write_lock_irqsave(&cpufreq_driver_lock, flags);
1064         for_each_cpu(j, policy->cpus)
1065                 per_cpu(cpufreq_cpu_data, j) = NULL;
1066         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1067
1068         kobject_put(&policy->kobj);
1069         wait_for_completion(&policy->kobj_unregister);
1070
1071 err_set_policy_cpu:
1072         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1073         free_cpumask_var(policy->related_cpus);
1074 err_free_cpumask:
1075         free_cpumask_var(policy->cpus);
1076 err_free_policy:
1077         kfree(policy);
1078 nomem_out:
1079         module_put(cpufreq_driver->owner);
1080 module_out:
1081         return ret;
1082 }
1083
1084 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1085 {
1086         int j;
1087
1088         policy->last_cpu = policy->cpu;
1089         policy->cpu = cpu;
1090
1091         for_each_cpu(j, policy->cpus)
1092                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1093
1094 #ifdef CONFIG_CPU_FREQ_TABLE
1095         cpufreq_frequency_table_update_policy_cpu(policy);
1096 #endif
1097         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1098                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1099 }
1100
1101 /**
1102  * __cpufreq_remove_dev - remove a CPU device
1103  *
1104  * Removes the cpufreq interface for a CPU device.
1105  * Caller should already have policy_rwsem in write mode for this CPU.
1106  * This routine frees the rwsem before returning.
1107  */
1108 static int __cpufreq_remove_dev(struct device *dev,
1109                 struct subsys_interface *sif)
1110 {
1111         unsigned int cpu = dev->id, ret, cpus;
1112         unsigned long flags;
1113         struct cpufreq_policy *data;
1114         struct kobject *kobj;
1115         struct completion *cmp;
1116         struct device *cpu_dev;
1117
1118         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1119
1120         write_lock_irqsave(&cpufreq_driver_lock, flags);
1121
1122         data = per_cpu(cpufreq_cpu_data, cpu);
1123         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1124
1125         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1126
1127         if (!data) {
1128                 pr_debug("%s: No cpu_data found\n", __func__);
1129                 return -EINVAL;
1130         }
1131
1132         if (cpufreq_driver->target)
1133                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1134
1135 #ifdef CONFIG_HOTPLUG_CPU
1136         if (!cpufreq_driver->setpolicy)
1137                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1138                         data->governor->name, CPUFREQ_NAME_LEN);
1139 #endif
1140
1141         WARN_ON(lock_policy_rwsem_write(cpu));
1142         cpus = cpumask_weight(data->cpus);
1143
1144         if (cpus > 1)
1145                 cpumask_clear_cpu(cpu, data->cpus);
1146         unlock_policy_rwsem_write(cpu);
1147
1148         if (cpu != data->cpu) {
1149                 sysfs_remove_link(&dev->kobj, "cpufreq");
1150         } else if (cpus > 1) {
1151                 /* first sibling now owns the new sysfs dir */
1152                 cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1153                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1154                 ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1155                 if (ret) {
1156                         pr_err("%s: Failed to move kobj: %d", __func__, ret);
1157
1158                         WARN_ON(lock_policy_rwsem_write(cpu));
1159                         cpumask_set_cpu(cpu, data->cpus);
1160
1161                         write_lock_irqsave(&cpufreq_driver_lock, flags);
1162                         per_cpu(cpufreq_cpu_data, cpu) = data;
1163                         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1164
1165                         unlock_policy_rwsem_write(cpu);
1166
1167                         ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1168                                         "cpufreq");
1169                         return -EINVAL;
1170                 }
1171
1172                 WARN_ON(lock_policy_rwsem_write(cpu));
1173                 update_policy_cpu(data, cpu_dev->id);
1174                 unlock_policy_rwsem_write(cpu);
1175                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1176                                 __func__, cpu_dev->id, cpu);
1177         }
1178
1179         if ((cpus == 1) && (cpufreq_driver->target))
1180                 __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1181
1182         pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1183         cpufreq_cpu_put(data);
1184
1185         /* If cpu is last user of policy, free policy */
1186         if (cpus == 1) {
1187                 lock_policy_rwsem_read(cpu);
1188                 kobj = &data->kobj;
1189                 cmp = &data->kobj_unregister;
1190                 unlock_policy_rwsem_read(cpu);
1191                 kobject_put(kobj);
1192
1193                 /* we need to make sure that the underlying kobj is actually
1194                  * not referenced anymore by anybody before we proceed with
1195                  * unloading.
1196                  */
1197                 pr_debug("waiting for dropping of refcount\n");
1198                 wait_for_completion(cmp);
1199                 pr_debug("wait complete\n");
1200
1201                 if (cpufreq_driver->exit)
1202                         cpufreq_driver->exit(data);
1203
1204                 free_cpumask_var(data->related_cpus);
1205                 free_cpumask_var(data->cpus);
1206                 kfree(data);
1207         } else if (cpufreq_driver->target) {
1208                 __cpufreq_governor(data, CPUFREQ_GOV_START);
1209                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1210         }
1211
1212         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1213         return 0;
1214 }
1215
1216 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1217 {
1218         unsigned int cpu = dev->id;
1219         int retval;
1220
1221         if (cpu_is_offline(cpu))
1222                 return 0;
1223
1224         retval = __cpufreq_remove_dev(dev, sif);
1225         return retval;
1226 }
1227
1228 static void handle_update(struct work_struct *work)
1229 {
1230         struct cpufreq_policy *policy =
1231                 container_of(work, struct cpufreq_policy, update);
1232         unsigned int cpu = policy->cpu;
1233         pr_debug("handle_update for cpu %u called\n", cpu);
1234         cpufreq_update_policy(cpu);
1235 }
1236
1237 /**
1238  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1239  *      in deep trouble.
1240  *      @cpu: cpu number
1241  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1242  *      @new_freq: CPU frequency the CPU actually runs at
1243  *
1244  *      We adjust to current frequency first, and need to clean up later.
1245  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1246  */
1247 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1248                                 unsigned int new_freq)
1249 {
1250         struct cpufreq_policy *policy;
1251         struct cpufreq_freqs freqs;
1252         unsigned long flags;
1253
1254         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1255                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1256
1257         freqs.old = old_freq;
1258         freqs.new = new_freq;
1259
1260         read_lock_irqsave(&cpufreq_driver_lock, flags);
1261         policy = per_cpu(cpufreq_cpu_data, cpu);
1262         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1263
1264         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1265         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1266 }
1267
1268 /**
1269  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1270  * @cpu: CPU number
1271  *
1272  * This is the last known freq, without actually getting it from the driver.
1273  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1274  */
1275 unsigned int cpufreq_quick_get(unsigned int cpu)
1276 {
1277         struct cpufreq_policy *policy;
1278         unsigned int ret_freq = 0;
1279
1280         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1281                 return cpufreq_driver->get(cpu);
1282
1283         policy = cpufreq_cpu_get(cpu);
1284         if (policy) {
1285                 ret_freq = policy->cur;
1286                 cpufreq_cpu_put(policy);
1287         }
1288
1289         return ret_freq;
1290 }
1291 EXPORT_SYMBOL(cpufreq_quick_get);
1292
1293 /**
1294  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1295  * @cpu: CPU number
1296  *
1297  * Just return the max possible frequency for a given CPU.
1298  */
1299 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1300 {
1301         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1302         unsigned int ret_freq = 0;
1303
1304         if (policy) {
1305                 ret_freq = policy->max;
1306                 cpufreq_cpu_put(policy);
1307         }
1308
1309         return ret_freq;
1310 }
1311 EXPORT_SYMBOL(cpufreq_quick_get_max);
1312
1313 static unsigned int __cpufreq_get(unsigned int cpu)
1314 {
1315         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1316         unsigned int ret_freq = 0;
1317
1318         if (!cpufreq_driver->get)
1319                 return ret_freq;
1320
1321         ret_freq = cpufreq_driver->get(cpu);
1322
1323         if (ret_freq && policy->cur &&
1324                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1325                 /* verify no discrepancy between actual and
1326                                         saved value exists */
1327                 if (unlikely(ret_freq != policy->cur)) {
1328                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1329                         schedule_work(&policy->update);
1330                 }
1331         }
1332
1333         return ret_freq;
1334 }
1335
1336 /**
1337  * cpufreq_get - get the current CPU frequency (in kHz)
1338  * @cpu: CPU number
1339  *
1340  * Get the CPU current (static) CPU frequency
1341  */
1342 unsigned int cpufreq_get(unsigned int cpu)
1343 {
1344         unsigned int ret_freq = 0;
1345         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1346
1347         if (!policy)
1348                 goto out;
1349
1350         if (unlikely(lock_policy_rwsem_read(cpu)))
1351                 goto out_policy;
1352
1353         ret_freq = __cpufreq_get(cpu);
1354
1355         unlock_policy_rwsem_read(cpu);
1356
1357 out_policy:
1358         cpufreq_cpu_put(policy);
1359 out:
1360         return ret_freq;
1361 }
1362 EXPORT_SYMBOL(cpufreq_get);
1363
1364 static struct subsys_interface cpufreq_interface = {
1365         .name           = "cpufreq",
1366         .subsys         = &cpu_subsys,
1367         .add_dev        = cpufreq_add_dev,
1368         .remove_dev     = cpufreq_remove_dev,
1369 };
1370
1371 /**
1372  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1373  *
1374  * This function is only executed for the boot processor.  The other CPUs
1375  * have been put offline by means of CPU hotplug.
1376  */
1377 static int cpufreq_bp_suspend(void)
1378 {
1379         int ret = 0;
1380
1381         int cpu = smp_processor_id();
1382         struct cpufreq_policy *cpu_policy;
1383
1384         pr_debug("suspending cpu %u\n", cpu);
1385
1386         /* If there's no policy for the boot CPU, we have nothing to do. */
1387         cpu_policy = cpufreq_cpu_get(cpu);
1388         if (!cpu_policy)
1389                 return 0;
1390
1391         if (cpufreq_driver->suspend) {
1392                 ret = cpufreq_driver->suspend(cpu_policy);
1393                 if (ret)
1394                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1395                                         "step on CPU %u\n", cpu_policy->cpu);
1396         }
1397
1398         cpufreq_cpu_put(cpu_policy);
1399         return ret;
1400 }
1401
1402 /**
1403  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1404  *
1405  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1406  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1407  *          restored. It will verify that the current freq is in sync with
1408  *          what we believe it to be. This is a bit later than when it
1409  *          should be, but nonethteless it's better than calling
1410  *          cpufreq_driver->get() here which might re-enable interrupts...
1411  *
1412  * This function is only executed for the boot CPU.  The other CPUs have not
1413  * been turned on yet.
1414  */
1415 static void cpufreq_bp_resume(void)
1416 {
1417         int ret = 0;
1418
1419         int cpu = smp_processor_id();
1420         struct cpufreq_policy *cpu_policy;
1421
1422         pr_debug("resuming cpu %u\n", cpu);
1423
1424         /* If there's no policy for the boot CPU, we have nothing to do. */
1425         cpu_policy = cpufreq_cpu_get(cpu);
1426         if (!cpu_policy)
1427                 return;
1428
1429         if (cpufreq_driver->resume) {
1430                 ret = cpufreq_driver->resume(cpu_policy);
1431                 if (ret) {
1432                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1433                                         "step on CPU %u\n", cpu_policy->cpu);
1434                         goto fail;
1435                 }
1436         }
1437
1438         schedule_work(&cpu_policy->update);
1439
1440 fail:
1441         cpufreq_cpu_put(cpu_policy);
1442 }
1443
1444 static struct syscore_ops cpufreq_syscore_ops = {
1445         .suspend        = cpufreq_bp_suspend,
1446         .resume         = cpufreq_bp_resume,
1447 };
1448
1449 /**
1450  *      cpufreq_get_current_driver - return current driver's name
1451  *
1452  *      Return the name string of the currently loaded cpufreq driver
1453  *      or NULL, if none.
1454  */
1455 const char *cpufreq_get_current_driver(void)
1456 {
1457         if (cpufreq_driver)
1458                 return cpufreq_driver->name;
1459
1460         return NULL;
1461 }
1462 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1463
1464 /*********************************************************************
1465  *                     NOTIFIER LISTS INTERFACE                      *
1466  *********************************************************************/
1467
1468 /**
1469  *      cpufreq_register_notifier - register a driver with cpufreq
1470  *      @nb: notifier function to register
1471  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1472  *
1473  *      Add a driver to one of two lists: either a list of drivers that
1474  *      are notified about clock rate changes (once before and once after
1475  *      the transition), or a list of drivers that are notified about
1476  *      changes in cpufreq policy.
1477  *
1478  *      This function may sleep, and has the same return conditions as
1479  *      blocking_notifier_chain_register.
1480  */
1481 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1482 {
1483         int ret;
1484
1485         if (cpufreq_disabled())
1486                 return -EINVAL;
1487
1488         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1489
1490         switch (list) {
1491         case CPUFREQ_TRANSITION_NOTIFIER:
1492                 ret = srcu_notifier_chain_register(
1493                                 &cpufreq_transition_notifier_list, nb);
1494                 break;
1495         case CPUFREQ_POLICY_NOTIFIER:
1496                 ret = blocking_notifier_chain_register(
1497                                 &cpufreq_policy_notifier_list, nb);
1498                 break;
1499         default:
1500                 ret = -EINVAL;
1501         }
1502
1503         return ret;
1504 }
1505 EXPORT_SYMBOL(cpufreq_register_notifier);
1506
1507 /**
1508  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1509  *      @nb: notifier block to be unregistered
1510  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1511  *
1512  *      Remove a driver from the CPU frequency notifier list.
1513  *
1514  *      This function may sleep, and has the same return conditions as
1515  *      blocking_notifier_chain_unregister.
1516  */
1517 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1518 {
1519         int ret;
1520
1521         if (cpufreq_disabled())
1522                 return -EINVAL;
1523
1524         switch (list) {
1525         case CPUFREQ_TRANSITION_NOTIFIER:
1526                 ret = srcu_notifier_chain_unregister(
1527                                 &cpufreq_transition_notifier_list, nb);
1528                 break;
1529         case CPUFREQ_POLICY_NOTIFIER:
1530                 ret = blocking_notifier_chain_unregister(
1531                                 &cpufreq_policy_notifier_list, nb);
1532                 break;
1533         default:
1534                 ret = -EINVAL;
1535         }
1536
1537         return ret;
1538 }
1539 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1540
1541
1542 /*********************************************************************
1543  *                              GOVERNORS                            *
1544  *********************************************************************/
1545
1546 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1547                             unsigned int target_freq,
1548                             unsigned int relation)
1549 {
1550         int retval = -EINVAL;
1551         unsigned int old_target_freq = target_freq;
1552
1553         if (cpufreq_disabled())
1554                 return -ENODEV;
1555         if (policy->transition_ongoing)
1556                 return -EBUSY;
1557
1558         /* Make sure that target_freq is within supported range */
1559         if (target_freq > policy->max)
1560                 target_freq = policy->max;
1561         if (target_freq < policy->min)
1562                 target_freq = policy->min;
1563
1564         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1565                         policy->cpu, target_freq, relation, old_target_freq);
1566
1567         if (target_freq == policy->cur)
1568                 return 0;
1569
1570         if (cpufreq_driver->target)
1571                 retval = cpufreq_driver->target(policy, target_freq, relation);
1572
1573         return retval;
1574 }
1575 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1576
1577 int cpufreq_driver_target(struct cpufreq_policy *policy,
1578                           unsigned int target_freq,
1579                           unsigned int relation)
1580 {
1581         int ret = -EINVAL;
1582
1583         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1584                 goto fail;
1585
1586         ret = __cpufreq_driver_target(policy, target_freq, relation);
1587
1588         unlock_policy_rwsem_write(policy->cpu);
1589
1590 fail:
1591         return ret;
1592 }
1593 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1594
1595 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1596 {
1597         if (cpufreq_disabled())
1598                 return 0;
1599
1600         if (!cpufreq_driver->getavg)
1601                 return 0;
1602
1603         return cpufreq_driver->getavg(policy, cpu);
1604 }
1605 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1606
1607 /*
1608  * when "event" is CPUFREQ_GOV_LIMITS
1609  */
1610
1611 static int __cpufreq_governor(struct cpufreq_policy *policy,
1612                                         unsigned int event)
1613 {
1614         int ret;
1615
1616         /* Only must be defined when default governor is known to have latency
1617            restrictions, like e.g. conservative or ondemand.
1618            That this is the case is already ensured in Kconfig
1619         */
1620 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1621         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1622 #else
1623         struct cpufreq_governor *gov = NULL;
1624 #endif
1625
1626         if (policy->governor->max_transition_latency &&
1627             policy->cpuinfo.transition_latency >
1628             policy->governor->max_transition_latency) {
1629                 if (!gov)
1630                         return -EINVAL;
1631                 else {
1632                         printk(KERN_WARNING "%s governor failed, too long"
1633                                " transition latency of HW, fallback"
1634                                " to %s governor\n",
1635                                policy->governor->name,
1636                                gov->name);
1637                         policy->governor = gov;
1638                 }
1639         }
1640
1641         if (!try_module_get(policy->governor->owner))
1642                 return -EINVAL;
1643
1644         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1645                                                 policy->cpu, event);
1646
1647         mutex_lock(&cpufreq_governor_lock);
1648         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1649             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1650                 mutex_unlock(&cpufreq_governor_lock);
1651                 return -EBUSY;
1652         }
1653
1654         if (event == CPUFREQ_GOV_STOP)
1655                 policy->governor_enabled = false;
1656         else if (event == CPUFREQ_GOV_START)
1657                 policy->governor_enabled = true;
1658
1659         mutex_unlock(&cpufreq_governor_lock);
1660
1661         ret = policy->governor->governor(policy, event);
1662
1663         if (!ret) {
1664                 if (event == CPUFREQ_GOV_POLICY_INIT)
1665                         policy->governor->initialized++;
1666                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1667                         policy->governor->initialized--;
1668         } else {
1669                 /* Restore original values */
1670                 mutex_lock(&cpufreq_governor_lock);
1671                 if (event == CPUFREQ_GOV_STOP)
1672                         policy->governor_enabled = true;
1673                 else if (event == CPUFREQ_GOV_START)
1674                         policy->governor_enabled = false;
1675                 mutex_unlock(&cpufreq_governor_lock);
1676         }
1677
1678         /* we keep one module reference alive for
1679                         each CPU governed by this CPU */
1680         if ((event != CPUFREQ_GOV_START) || ret)
1681                 module_put(policy->governor->owner);
1682         if ((event == CPUFREQ_GOV_STOP) && !ret)
1683                 module_put(policy->governor->owner);
1684
1685         return ret;
1686 }
1687
1688 int cpufreq_register_governor(struct cpufreq_governor *governor)
1689 {
1690         int err;
1691
1692         if (!governor)
1693                 return -EINVAL;
1694
1695         if (cpufreq_disabled())
1696                 return -ENODEV;
1697
1698         mutex_lock(&cpufreq_governor_mutex);
1699
1700         governor->initialized = 0;
1701         err = -EBUSY;
1702         if (__find_governor(governor->name) == NULL) {
1703                 err = 0;
1704                 list_add(&governor->governor_list, &cpufreq_governor_list);
1705         }
1706
1707         mutex_unlock(&cpufreq_governor_mutex);
1708         return err;
1709 }
1710 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1711
1712 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1713 {
1714 #ifdef CONFIG_HOTPLUG_CPU
1715         int cpu;
1716 #endif
1717
1718         if (!governor)
1719                 return;
1720
1721         if (cpufreq_disabled())
1722                 return;
1723
1724 #ifdef CONFIG_HOTPLUG_CPU
1725         for_each_present_cpu(cpu) {
1726                 if (cpu_online(cpu))
1727                         continue;
1728                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1729                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1730         }
1731 #endif
1732
1733         mutex_lock(&cpufreq_governor_mutex);
1734         list_del(&governor->governor_list);
1735         mutex_unlock(&cpufreq_governor_mutex);
1736         return;
1737 }
1738 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1739
1740
1741 /*********************************************************************
1742  *                          POLICY INTERFACE                         *
1743  *********************************************************************/
1744
1745 /**
1746  * cpufreq_get_policy - get the current cpufreq_policy
1747  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1748  *      is written
1749  *
1750  * Reads the current cpufreq policy.
1751  */
1752 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1753 {
1754         struct cpufreq_policy *cpu_policy;
1755         if (!policy)
1756                 return -EINVAL;
1757
1758         cpu_policy = cpufreq_cpu_get(cpu);
1759         if (!cpu_policy)
1760                 return -EINVAL;
1761
1762         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1763
1764         cpufreq_cpu_put(cpu_policy);
1765         return 0;
1766 }
1767 EXPORT_SYMBOL(cpufreq_get_policy);
1768
1769 /*
1770  * data   : current policy.
1771  * policy : policy to be set.
1772  */
1773 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1774                                 struct cpufreq_policy *policy)
1775 {
1776         int ret = 0, failed = 1;
1777
1778         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1779                 policy->min, policy->max);
1780
1781         memcpy(&policy->cpuinfo, &data->cpuinfo,
1782                                 sizeof(struct cpufreq_cpuinfo));
1783
1784         if (policy->min > data->max || policy->max < data->min) {
1785                 ret = -EINVAL;
1786                 goto error_out;
1787         }
1788
1789         /* verify the cpu speed can be set within this limit */
1790         ret = cpufreq_driver->verify(policy);
1791         if (ret)
1792                 goto error_out;
1793
1794         /* adjust if necessary - all reasons */
1795         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1796                         CPUFREQ_ADJUST, policy);
1797
1798         /* adjust if necessary - hardware incompatibility*/
1799         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1800                         CPUFREQ_INCOMPATIBLE, policy);
1801
1802         /*
1803          * verify the cpu speed can be set within this limit, which might be
1804          * different to the first one
1805          */
1806         ret = cpufreq_driver->verify(policy);
1807         if (ret)
1808                 goto error_out;
1809
1810         /* notification of the new policy */
1811         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1812                         CPUFREQ_NOTIFY, policy);
1813
1814         data->min = policy->min;
1815         data->max = policy->max;
1816
1817         pr_debug("new min and max freqs are %u - %u kHz\n",
1818                                         data->min, data->max);
1819
1820         if (cpufreq_driver->setpolicy) {
1821                 data->policy = policy->policy;
1822                 pr_debug("setting range\n");
1823                 ret = cpufreq_driver->setpolicy(policy);
1824         } else {
1825                 if (policy->governor != data->governor) {
1826                         /* save old, working values */
1827                         struct cpufreq_governor *old_gov = data->governor;
1828
1829                         pr_debug("governor switch\n");
1830
1831                         /* end old governor */
1832                         if (data->governor) {
1833                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1834                                 unlock_policy_rwsem_write(policy->cpu);
1835                                 __cpufreq_governor(data,
1836                                                 CPUFREQ_GOV_POLICY_EXIT);
1837                                 lock_policy_rwsem_write(policy->cpu);
1838                         }
1839
1840                         /* start new governor */
1841                         data->governor = policy->governor;
1842                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1843                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1844                                         failed = 0;
1845                                 } else {
1846                                         unlock_policy_rwsem_write(policy->cpu);
1847                                         __cpufreq_governor(data,
1848                                                         CPUFREQ_GOV_POLICY_EXIT);
1849                                         lock_policy_rwsem_write(policy->cpu);
1850                                 }
1851                         }
1852
1853                         if (failed) {
1854                                 /* new governor failed, so re-start old one */
1855                                 pr_debug("starting governor %s failed\n",
1856                                                         data->governor->name);
1857                                 if (old_gov) {
1858                                         data->governor = old_gov;
1859                                         __cpufreq_governor(data,
1860                                                         CPUFREQ_GOV_POLICY_INIT);
1861                                         __cpufreq_governor(data,
1862                                                            CPUFREQ_GOV_START);
1863                                 }
1864                                 ret = -EINVAL;
1865                                 goto error_out;
1866                         }
1867                         /* might be a policy change, too, so fall through */
1868                 }
1869                 pr_debug("governor: change or update limits\n");
1870                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1871         }
1872
1873 error_out:
1874         return ret;
1875 }
1876
1877 /**
1878  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1879  *      @cpu: CPU which shall be re-evaluated
1880  *
1881  *      Useful for policy notifiers which have different necessities
1882  *      at different times.
1883  */
1884 int cpufreq_update_policy(unsigned int cpu)
1885 {
1886         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1887         struct cpufreq_policy policy;
1888         int ret;
1889
1890         if (!data) {
1891                 ret = -ENODEV;
1892                 goto no_policy;
1893         }
1894
1895         if (unlikely(lock_policy_rwsem_write(cpu))) {
1896                 ret = -EINVAL;
1897                 goto fail;
1898         }
1899
1900         pr_debug("updating policy for CPU %u\n", cpu);
1901         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1902         policy.min = data->user_policy.min;
1903         policy.max = data->user_policy.max;
1904         policy.policy = data->user_policy.policy;
1905         policy.governor = data->user_policy.governor;
1906
1907         /*
1908          * BIOS might change freq behind our back
1909          * -> ask driver for current freq and notify governors about a change
1910          */
1911         if (cpufreq_driver->get) {
1912                 policy.cur = cpufreq_driver->get(cpu);
1913                 if (!data->cur) {
1914                         pr_debug("Driver did not initialize current freq");
1915                         data->cur = policy.cur;
1916                 } else {
1917                         if (data->cur != policy.cur && cpufreq_driver->target)
1918                                 cpufreq_out_of_sync(cpu, data->cur,
1919                                                                 policy.cur);
1920                 }
1921         }
1922
1923         ret = __cpufreq_set_policy(data, &policy);
1924
1925         unlock_policy_rwsem_write(cpu);
1926
1927 fail:
1928         cpufreq_cpu_put(data);
1929 no_policy:
1930         return ret;
1931 }
1932 EXPORT_SYMBOL(cpufreq_update_policy);
1933
1934 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1935                                         unsigned long action, void *hcpu)
1936 {
1937         unsigned int cpu = (unsigned long)hcpu;
1938         struct device *dev;
1939
1940         dev = get_cpu_device(cpu);
1941         if (dev) {
1942                 switch (action) {
1943                 case CPU_ONLINE:
1944                         cpufreq_add_dev(dev, NULL);
1945                         break;
1946                 case CPU_DOWN_PREPARE:
1947                 case CPU_UP_CANCELED_FROZEN:
1948                         __cpufreq_remove_dev(dev, NULL);
1949                         break;
1950                 case CPU_DOWN_FAILED:
1951                         cpufreq_add_dev(dev, NULL);
1952                         break;
1953                 }
1954         }
1955         return NOTIFY_OK;
1956 }
1957
1958 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1959         .notifier_call = cpufreq_cpu_callback,
1960 };
1961
1962 /*********************************************************************
1963  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1964  *********************************************************************/
1965
1966 /**
1967  * cpufreq_register_driver - register a CPU Frequency driver
1968  * @driver_data: A struct cpufreq_driver containing the values#
1969  * submitted by the CPU Frequency driver.
1970  *
1971  * Registers a CPU Frequency driver to this core code. This code
1972  * returns zero on success, -EBUSY when another driver got here first
1973  * (and isn't unregistered in the meantime).
1974  *
1975  */
1976 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1977 {
1978         unsigned long flags;
1979         int ret;
1980
1981         if (cpufreq_disabled())
1982                 return -ENODEV;
1983
1984         if (!driver_data || !driver_data->verify || !driver_data->init ||
1985             ((!driver_data->setpolicy) && (!driver_data->target)))
1986                 return -EINVAL;
1987
1988         pr_debug("trying to register driver %s\n", driver_data->name);
1989
1990         if (driver_data->setpolicy)
1991                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1992
1993         write_lock_irqsave(&cpufreq_driver_lock, flags);
1994         if (cpufreq_driver) {
1995                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1996                 return -EBUSY;
1997         }
1998         cpufreq_driver = driver_data;
1999         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2000
2001         ret = subsys_interface_register(&cpufreq_interface);
2002         if (ret)
2003                 goto err_null_driver;
2004
2005         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2006                 int i;
2007                 ret = -ENODEV;
2008
2009                 /* check for at least one working CPU */
2010                 for (i = 0; i < nr_cpu_ids; i++)
2011                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2012                                 ret = 0;
2013                                 break;
2014                         }
2015
2016                 /* if all ->init() calls failed, unregister */
2017                 if (ret) {
2018                         pr_debug("no CPU initialized for driver %s\n",
2019                                                         driver_data->name);
2020                         goto err_if_unreg;
2021                 }
2022         }
2023
2024         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2025         pr_debug("driver %s up and running\n", driver_data->name);
2026
2027         return 0;
2028 err_if_unreg:
2029         subsys_interface_unregister(&cpufreq_interface);
2030 err_null_driver:
2031         write_lock_irqsave(&cpufreq_driver_lock, flags);
2032         cpufreq_driver = NULL;
2033         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2034         return ret;
2035 }
2036 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2037
2038 /**
2039  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2040  *
2041  * Unregister the current CPUFreq driver. Only call this if you have
2042  * the right to do so, i.e. if you have succeeded in initialising before!
2043  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2044  * currently not initialised.
2045  */
2046 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2047 {
2048         unsigned long flags;
2049
2050         if (!cpufreq_driver || (driver != cpufreq_driver))
2051                 return -EINVAL;
2052
2053         pr_debug("unregistering driver %s\n", driver->name);
2054
2055         subsys_interface_unregister(&cpufreq_interface);
2056         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2057
2058         write_lock_irqsave(&cpufreq_driver_lock, flags);
2059         cpufreq_driver = NULL;
2060         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2061
2062         return 0;
2063 }
2064 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2065
2066 static int __init cpufreq_core_init(void)
2067 {
2068         int cpu;
2069
2070         if (cpufreq_disabled())
2071                 return -ENODEV;
2072
2073         for_each_possible_cpu(cpu) {
2074                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2075                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2076         }
2077
2078         cpufreq_global_kobject = kobject_create();
2079         BUG_ON(!cpufreq_global_kobject);
2080         register_syscore_ops(&cpufreq_syscore_ops);
2081
2082         return 0;
2083 }
2084 core_initcall(cpufreq_core_init);