drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy,
 108                 unsigned int event);
 109 static unsigned int __cpufreq_get(unsigned int cpu);
 110 static void handle_update(struct work_struct *work);
 111
 112 /**
 113  * Two notifier lists: the "policy" list is involved in the
 114  * validation process for a new CPU frequency policy; the
 115  * "transition" list for kernel code that needs to handle
 116  * changes to devices when the CPU clock speed changes.
 117  * The mutex locks both lists.
 118  */
 119 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 120 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 121
 122 static bool init_cpufreq_transition_notifier_list_called;
 123 static int __init init_cpufreq_transition_notifier_list(void)
 124 {
 125         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 126         init_cpufreq_transition_notifier_list_called = true;
 127         return 0;
 128 }
 129 pure_initcall(init_cpufreq_transition_notifier_list);
 130
 131 static LIST_HEAD(cpufreq_governor_list);
 132 static DEFINE_MUTEX(cpufreq_governor_mutex);
 133
 134 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 135 {
 136         struct cpufreq_policy *data;
 137         unsigned long flags;
 138
 139         if (cpu >= nr_cpu_ids)
 140                 goto err_out;
 141
 142         /* get the cpufreq driver */
 143         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 144
 145         if (!cpufreq_driver)
 146                 goto err_out_unlock;
 147
 148         if (!try_module_get(cpufreq_driver->owner))
 149                 goto err_out_unlock;
 150
 151
 152         /* get the CPU */
 153         data = per_cpu(cpufreq_cpu_data, cpu);
 154
 155         if (!data)
 156                 goto err_out_put_module;
 157
 158         if (!kobject_get(&data->kobj))
 159                 goto err_out_put_module;
 160
 161         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 162         return data;
 163
 164 err_out_put_module:
 165         module_put(cpufreq_driver->owner);
 166 err_out_unlock:
 167         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 168 err_out:
 169         return NULL;
 170 }
 171 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 172
 173
 174 void cpufreq_cpu_put(struct cpufreq_policy *data)
 175 {
 176         kobject_put(&data->kobj);
 177         module_put(cpufreq_driver->owner);
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 180
 181
 182 /*********************************************************************
 183  *                     UNIFIED DEBUG HELPERS                         *
 184  *********************************************************************/
 185 #ifdef CONFIG_CPU_FREQ_DEBUG
 186
 187 /* what part(s) of the CPUfreq subsystem are debugged? */
 188 static unsigned int debug;
 189
 190 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 191  * set or modify this value.
 192  */
 193 static unsigned int debug_ratelimit = 1;
 194
 195 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 196  * loading of a cpufreq driver, temporarily disabled when a new policy
 197  * is set, and disabled upon cpufreq driver removal
 198  */
 199 static unsigned int disable_ratelimit = 1;
 200 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 201
 202 static void cpufreq_debug_enable_ratelimit(void)
 203 {
 204         unsigned long flags;
 205
 206         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 207         if (disable_ratelimit)
 208                 disable_ratelimit--;
 209         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 210 }
 211
 212 static void cpufreq_debug_disable_ratelimit(void)
 213 {
 214         unsigned long flags;
 215
 216         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 217         disable_ratelimit++;
 218         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 219 }
 220
 221 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 222                         const char *fmt, ...)
 223 {
 224         char s[256];
 225         va_list args;
 226         unsigned int len;
 227         unsigned long flags;
 228
 229         WARN_ON(!prefix);
 230         if (type & debug) {
 231                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 232                 if (!disable_ratelimit && debug_ratelimit
 233                                         && !printk_ratelimit()) {
 234                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235                         return;
 236                 }
 237                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 238
 239                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 240
 241                 va_start(args, fmt);
 242                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 243                 va_end(args);
 244
 245                 printk(s);
 246
 247                 WARN_ON(len < 5);
 248         }
 249 }
 250 EXPORT_SYMBOL(cpufreq_debug_printk);
 251
 252
 253 module_param(debug, uint, 0644);
 254 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 255                         " 2 to debug drivers, and 4 to debug governors.");
 256
 257 module_param(debug_ratelimit, uint, 0644);
 258 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 259                                         " set to 0 to disable ratelimiting.");
 260
 261 #else /* !CONFIG_CPU_FREQ_DEBUG */
 262
 263 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 264 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 265
 266 #endif /* CONFIG_CPU_FREQ_DEBUG */
 267
 268
 269 /*********************************************************************
 270  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 271  *********************************************************************/
 272
 273 /**
 274  * adjust_jiffies - adjust the system "loops_per_jiffy"
 275  *
 276  * This function alters the system "loops_per_jiffy" for the clock
 277  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 278  * systems as each CPU might be scaled differently. So, use the arch
 279  * per-CPU loops_per_jiffy value wherever possible.
 280  */
 281 #ifndef CONFIG_SMP
 282 static unsigned long l_p_j_ref;
 283 static unsigned int  l_p_j_ref_freq;
 284
 285 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 286 {
 287         if (ci->flags & CPUFREQ_CONST_LOOPS)
 288                 return;
 289
 290         if (!l_p_j_ref_freq) {
 291                 l_p_j_ref = loops_per_jiffy;
 292                 l_p_j_ref_freq = ci->old;
 293                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 294                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 295         }
 296         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 297             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 298             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 299                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 300                                                                 ci->new);
 301                 dprintk("scaling loops_per_jiffy to %lu "
 302                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 303         }
 304 }
 305 #else
 306 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 307 {
 308         return;
 309 }
 310 #endif
 311
 312
 313 /**
 314  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 315  * on frequency transition.
 316  *
 317  * This function calls the transition notifiers and the "adjust_jiffies"
 318  * function. It is called twice on all CPU frequency changes that have
 319  * external effects.
 320  */
 321 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 322 {
 323         struct cpufreq_policy *policy;
 324
 325         BUG_ON(irqs_disabled());
 326
 327         freqs->flags = cpufreq_driver->flags;
 328         dprintk("notification %u of frequency transition to %u kHz\n",
 329                 state, freqs->new);
 330
 331         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 332         switch (state) {
 333
 334         case CPUFREQ_PRECHANGE:
 335                 /* detect if the driver reported a value as "old frequency"
 336                  * which is not equal to what the cpufreq core thinks is
 337                  * "old frequency".
 338                  */
 339                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 340                         if ((policy) && (policy->cpu == freqs->cpu) &&
 341                             (policy->cur) && (policy->cur != freqs->old)) {
 342                                 dprintk("Warning: CPU frequency is"
 343                                         " %u, cpufreq assumed %u kHz.\n",
 344                                         freqs->old, policy->cur);
 345                                 freqs->old = policy->cur;
 346                         }
 347                 }
 348                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 349                                 CPUFREQ_PRECHANGE, freqs);
 350                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 351                 break;
 352
 353         case CPUFREQ_POSTCHANGE:
 354                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 355                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 356                                 CPUFREQ_POSTCHANGE, freqs);
 357                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 358                         policy->cur = freqs->new;
 359                 break;
 360         }
 361 }
 362 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 363
 364
 365
 366 /*********************************************************************
 367  *                          SYSFS INTERFACE                          *
 368  *********************************************************************/
 369
 370 static struct cpufreq_governor *__find_governor(const char *str_governor)
 371 {
 372         struct cpufreq_governor *t;
 373
 374         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 375                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 376                         return t;
 377
 378         return NULL;
 379 }
 380
 381 /**
 382  * cpufreq_parse_governor - parse a governor string
 383  */
 384 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 385                                 struct cpufreq_governor **governor)
 386 {
 387         int err = -EINVAL;
 388
 389         if (!cpufreq_driver)
 390                 goto out;
 391
 392         if (cpufreq_driver->setpolicy) {
 393                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 394                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 395                         err = 0;
 396                 } else if (!strnicmp(str_governor, "powersave",
 397                                                 CPUFREQ_NAME_LEN)) {
 398                         *policy = CPUFREQ_POLICY_POWERSAVE;
 399                         err = 0;
 400                 }
 401         } else if (cpufreq_driver->target) {
 402                 struct cpufreq_governor *t;
 403
 404                 mutex_lock(&cpufreq_governor_mutex);
 405
 406                 t = __find_governor(str_governor);
 407
 408                 if (t == NULL) {
 409                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 410                                                                 str_governor);
 411
 412                         if (name) {
 413                                 int ret;
 414
 415                                 mutex_unlock(&cpufreq_governor_mutex);
 416                                 ret = request_module("%s", name);
 417                                 mutex_lock(&cpufreq_governor_mutex);
 418
 419                                 if (ret == 0)
 420                                         t = __find_governor(str_governor);
 421                         }
 422
 423                         kfree(name);
 424                 }
 425
 426                 if (t != NULL) {
 427                         *governor = t;
 428                         err = 0;
 429                 }
 430
 431                 mutex_unlock(&cpufreq_governor_mutex);
 432         }
 433 out:
 434         return err;
 435 }
 436
 437
 438 /**
 439  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 440  * print out cpufreq information
 441  *
 442  * Write out information from cpufreq_driver->policy[cpu]; object must be
 443  * "unsigned int".
 444  */
 445
 446 #define show_one(file_name, object)                     \
 447 static ssize_t show_##file_name                         \
 448 (struct cpufreq_policy *policy, char *buf)              \
 449 {                                                       \
 450         return sprintf(buf, "%u\n", policy->object);    \
 451 }
 452
 453 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 454 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 455 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf(buf, "%u", &new_policy.object);                    \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq, min);
 488 store_one(scaling_max_freq, max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 494                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 507 {
 508         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 509                 return sprintf(buf, "powersave\n");
 510         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 511                 return sprintf(buf, "performance\n");
 512         else if (policy->governor)
 513                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 514                                 policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 566                                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 578                     - (CPUFREQ_NAME_LEN + 2)))
 579                         goto out;
 580                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 581         }
 582 out:
 583         i += sprintf(&buf[i], "\n");
 584         return i;
 585 }
 586
 587 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 588 {
 589         ssize_t i = 0;
 590         unsigned int cpu;
 591
 592         for_each_cpu(cpu, mask) {
 593                 if (i)
 594                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 595                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 596                 if (i >= (PAGE_SIZE - 5))
 597                         break;
 598         }
 599         i += sprintf(&buf[i], "\n");
 600         return i;
 601 }
 602
 603 /**
 604  * show_related_cpus - show the CPUs affected by each transition even if
 605  * hw coordination is in use
 606  */
 607 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 608 {
 609         if (cpumask_empty(policy->related_cpus))
 610                 return show_cpus(policy->cpus, buf);
 611         return show_cpus(policy->related_cpus, buf);
 612 }
 613
 614 /**
 615  * show_affected_cpus - show the CPUs affected by each transition
 616  */
 617 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 618 {
 619         return show_cpus(policy->cpus, buf);
 620 }
 621
 622 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 623                                         const char *buf, size_t count)
 624 {
 625         unsigned int freq = 0;
 626         unsigned int ret;
 627
 628         if (!policy->governor || !policy->governor->store_setspeed)
 629                 return -EINVAL;
 630
 631         ret = sscanf(buf, "%u", &freq);
 632         if (ret != 1)
 633                 return -EINVAL;
 634
 635         policy->governor->store_setspeed(policy, freq);
 636
 637         return count;
 638 }
 639
 640 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 641 {
 642         if (!policy->governor || !policy->governor->show_setspeed)
 643                 return sprintf(buf, "<unsupported>\n");
 644
 645         return policy->governor->show_setspeed(policy, buf);
 646 }
 647
 648 #define define_one_ro(_name) \
 649 static struct freq_attr _name = \
 650 __ATTR(_name, 0444, show_##_name, NULL)
 651
 652 #define define_one_ro0400(_name) \
 653 static struct freq_attr _name = \
 654 __ATTR(_name, 0400, show_##_name, NULL)
 655
 656 #define define_one_rw(_name) \
 657 static struct freq_attr _name = \
 658 __ATTR(_name, 0644, show_##_name, store_##_name)
 659
 660 define_one_ro0400(cpuinfo_cur_freq);
 661 define_one_ro(cpuinfo_min_freq);
 662 define_one_ro(cpuinfo_max_freq);
 663 define_one_ro(cpuinfo_transition_latency);
 664 define_one_ro(scaling_available_governors);
 665 define_one_ro(scaling_driver);
 666 define_one_ro(scaling_cur_freq);
 667 define_one_ro(related_cpus);
 668 define_one_ro(affected_cpus);
 669 define_one_rw(scaling_min_freq);
 670 define_one_rw(scaling_max_freq);
 671 define_one_rw(scaling_governor);
 672 define_one_rw(scaling_setspeed);
 673
 674 static struct attribute *default_attrs[] = {
 675         &cpuinfo_min_freq.attr,
 676         &cpuinfo_max_freq.attr,
 677         &cpuinfo_transition_latency.attr,
 678         &scaling_min_freq.attr,
 679         &scaling_max_freq.attr,
 680         &affected_cpus.attr,
 681         &related_cpus.attr,
 682         &scaling_governor.attr,
 683         &scaling_driver.attr,
 684         &scaling_available_governors.attr,
 685         &scaling_setspeed.attr,
 686         NULL
 687 };
 688
 689 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 690 #define to_attr(a) container_of(a, struct freq_attr, attr)
 691
 692 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 693 {
 694         struct cpufreq_policy *policy = to_policy(kobj);
 695         struct freq_attr *fattr = to_attr(attr);
 696         ssize_t ret = -EINVAL;
 697         policy = cpufreq_cpu_get(policy->cpu);
 698         if (!policy)
 699                 goto no_policy;
 700
 701         if (lock_policy_rwsem_read(policy->cpu) < 0)
 702                 goto fail;
 703
 704         if (fattr->show)
 705                 ret = fattr->show(policy, buf);
 706         else
 707                 ret = -EIO;
 708
 709         unlock_policy_rwsem_read(policy->cpu);
 710 fail:
 711         cpufreq_cpu_put(policy);
 712 no_policy:
 713         return ret;
 714 }
 715
 716 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 717                      const char *buf, size_t count)
 718 {
 719         struct cpufreq_policy *policy = to_policy(kobj);
 720         struct freq_attr *fattr = to_attr(attr);
 721         ssize_t ret = -EINVAL;
 722         policy = cpufreq_cpu_get(policy->cpu);
 723         if (!policy)
 724                 goto no_policy;
 725
 726         if (lock_policy_rwsem_write(policy->cpu) < 0)
 727                 goto fail;
 728
 729         if (fattr->store)
 730                 ret = fattr->store(policy, buf, count);
 731         else
 732                 ret = -EIO;
 733
 734         unlock_policy_rwsem_write(policy->cpu);
 735 fail:
 736         cpufreq_cpu_put(policy);
 737 no_policy:
 738         return ret;
 739 }
 740
 741 static void cpufreq_sysfs_release(struct kobject *kobj)
 742 {
 743         struct cpufreq_policy *policy = to_policy(kobj);
 744         dprintk("last reference is dropped\n");
 745         complete(&policy->kobj_unregister);
 746 }
 747
 748 static struct sysfs_ops sysfs_ops = {
 749         .show   = show,
 750         .store  = store,
 751 };
 752
 753 static struct kobj_type ktype_cpufreq = {
 754         .sysfs_ops      = &sysfs_ops,
 755         .default_attrs  = default_attrs,
 756         .release        = cpufreq_sysfs_release,
 757 };
 758
 759
 760 /**
 761  * cpufreq_add_dev - add a CPU device
 762  *
 763  * Adds the cpufreq interface for a CPU device.
 764  */
 765 static int cpufreq_add_dev(struct sys_device *sys_dev)
 766 {
 767         unsigned int cpu = sys_dev->id;
 768         int ret = 0;
 769         struct cpufreq_policy new_policy;
 770         struct cpufreq_policy *policy;
 771         struct freq_attr **drv_attr;
 772         struct sys_device *cpu_sys_dev;
 773         unsigned long flags;
 774         unsigned int j;
 775 #ifdef CONFIG_SMP
 776         struct cpufreq_policy *managed_policy;
 777 #endif
 778
 779         if (cpu_is_offline(cpu))
 780                 return 0;
 781
 782         cpufreq_debug_disable_ratelimit();
 783         dprintk("adding CPU %u\n", cpu);
 784
 785 #ifdef CONFIG_SMP
 786         /* check whether a different CPU already registered this
 787          * CPU because it is in the same boat. */
 788         policy = cpufreq_cpu_get(cpu);
 789         if (unlikely(policy)) {
 790                 cpufreq_cpu_put(policy);
 791                 cpufreq_debug_enable_ratelimit();
 792                 return 0;
 793         }
 794 #endif
 795
 796         if (!try_module_get(cpufreq_driver->owner)) {
 797                 ret = -EINVAL;
 798                 goto module_out;
 799         }
 800
 801         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 802         if (!policy) {
 803                 ret = -ENOMEM;
 804                 goto nomem_out;
 805         }
 806         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) {
 807                 kfree(policy);
 808                 ret = -ENOMEM;
 809                 goto nomem_out;
 810         }
 811         if (!alloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) {
 812                 free_cpumask_var(policy->cpus);
 813                 kfree(policy);
 814                 ret = -ENOMEM;
 815                 goto nomem_out;
 816         }
 817
 818         policy->cpu = cpu;
 819         cpumask_copy(policy->cpus, cpumask_of(cpu));
 820
 821         /* Initially set CPU itself as the policy_cpu */
 822         per_cpu(policy_cpu, cpu) = cpu;
 823         lock_policy_rwsem_write(cpu);
 824
 825         init_completion(&policy->kobj_unregister);
 826         INIT_WORK(&policy->update, handle_update);
 827
 828         /* Set governor before ->init, so that driver could check it */
 829         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 830         /* call driver. From then on the cpufreq must be able
 831          * to accept all calls to ->verify and ->setpolicy for this CPU
 832          */
 833         ret = cpufreq_driver->init(policy);
 834         if (ret) {
 835                 dprintk("initialization failed\n");
 836                 goto err_out;
 837         }
 838         policy->user_policy.min = policy->min;
 839         policy->user_policy.max = policy->max;
 840
 841         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 842                                      CPUFREQ_START, policy);
 843
 844 #ifdef CONFIG_SMP
 845
 846 #ifdef CONFIG_HOTPLUG_CPU
 847         if (per_cpu(cpufreq_cpu_governor, cpu)) {
 848                 policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 849                 dprintk("Restoring governor %s for cpu %d\n",
 850                        policy->governor->name, cpu);
 851         }
 852 #endif
 853
 854         for_each_cpu(j, policy->cpus) {
 855                 if (cpu == j)
 856                         continue;
 857
 858                 /* Check for existing affected CPUs.
 859                  * They may not be aware of it due to CPU Hotplug.
 860                  */
 861                 managed_policy = cpufreq_cpu_get(j);            /* FIXME: Where is this released?  What about error paths? */
 862                 if (unlikely(managed_policy)) {
 863
 864                         /* Set proper policy_cpu */
 865                         unlock_policy_rwsem_write(cpu);
 866                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 867
 868                         if (lock_policy_rwsem_write(cpu) < 0)
 869                                 goto err_out_driver_exit;
 870
 871                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 872                         cpumask_copy(managed_policy->cpus, policy->cpus);
 873                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 874                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 875
 876                         dprintk("CPU already managed, adding link\n");
 877                         ret = sysfs_create_link(&sys_dev->kobj,
 878                                                 &managed_policy->kobj,
 879                                                 "cpufreq");
 880                         if (ret)
 881                                 goto err_out_driver_exit;
 882
 883                         cpufreq_debug_enable_ratelimit();
 884                         ret = 0;
 885                         goto err_out_driver_exit; /* call driver->exit() */
 886                 }
 887         }
 888 #endif
 889         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 890
 891         /* prepare interface data */
 892         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, &sys_dev->kobj,
 893                                    "cpufreq");
 894         if (ret)
 895                 goto err_out_driver_exit;
 896
 897         /* set up files for this cpu device */
 898         drv_attr = cpufreq_driver->attr;
 899         while ((drv_attr) && (*drv_attr)) {
 900                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 901                 if (ret)
 902                         goto err_out_driver_exit;
 903                 drv_attr++;
 904         }
 905         if (cpufreq_driver->get) {
 906                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 907                 if (ret)
 908                         goto err_out_driver_exit;
 909         }
 910         if (cpufreq_driver->target) {
 911                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 912                 if (ret)
 913                         goto err_out_driver_exit;
 914         }
 915
 916         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 917         for_each_cpu(j, policy->cpus) {
 918                 per_cpu(cpufreq_cpu_data, j) = policy;
 919                 per_cpu(policy_cpu, j) = policy->cpu;
 920         }
 921         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 922
 923         /* symlink affected CPUs */
 924         for_each_cpu(j, policy->cpus) {
 925                 if (j == cpu)
 926                         continue;
 927                 if (!cpu_online(j))
 928                         continue;
 929
 930                 dprintk("CPU %u already managed, adding link\n", j);
 931                 cpufreq_cpu_get(cpu);
 932                 cpu_sys_dev = get_cpu_sysdev(j);
 933                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 934                                         "cpufreq");
 935                 if (ret)
 936                         goto err_out_unregister;
 937         }
 938
 939         policy->governor = NULL; /* to assure that the starting sequence is
 940                                   * run in cpufreq_set_policy */
 941
 942         /* set default policy */
 943         ret = __cpufreq_set_policy(policy, &new_policy);
 944         policy->user_policy.policy = policy->policy;
 945         policy->user_policy.governor = policy->governor;
 946
 947         if (ret) {
 948                 dprintk("setting policy failed\n");
 949                 goto err_out_unregister;
 950         }
 951
 952         unlock_policy_rwsem_write(cpu);
 953
 954         kobject_uevent(&policy->kobj, KOBJ_ADD);
 955         module_put(cpufreq_driver->owner);
 956         dprintk("initialization complete\n");
 957         cpufreq_debug_enable_ratelimit();
 958
 959         return 0;
 960
 961
 962 err_out_unregister:
 963         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 964         for_each_cpu(j, policy->cpus)
 965                 per_cpu(cpufreq_cpu_data, j) = NULL;
 966         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 967
 968         kobject_put(&policy->kobj);
 969         wait_for_completion(&policy->kobj_unregister);
 970
 971 err_out_driver_exit:
 972         if (cpufreq_driver->exit)
 973                 cpufreq_driver->exit(policy);
 974
 975 err_out:
 976         unlock_policy_rwsem_write(cpu);
 977         kfree(policy);
 978
 979 nomem_out:
 980         module_put(cpufreq_driver->owner);
 981 module_out:
 982         cpufreq_debug_enable_ratelimit();
 983         return ret;
 984 }
 985
 986
 987 /**
 988  * __cpufreq_remove_dev - remove a CPU device
 989  *
 990  * Removes the cpufreq interface for a CPU device.
 991  * Caller should already have policy_rwsem in write mode for this CPU.
 992  * This routine frees the rwsem before returning.
 993  */
 994 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
 995 {
 996         unsigned int cpu = sys_dev->id;
 997         unsigned long flags;
 998         struct cpufreq_policy *data;
 999 #ifdef CONFIG_SMP
1000         struct sys_device *cpu_sys_dev;
1001         unsigned int j;
1002 #endif
1003
1004         cpufreq_debug_disable_ratelimit();
1005         dprintk("unregistering CPU %u\n", cpu);
1006
1007         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1008         data = per_cpu(cpufreq_cpu_data, cpu);
1009
1010         if (!data) {
1011                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1012                 cpufreq_debug_enable_ratelimit();
1013                 unlock_policy_rwsem_write(cpu);
1014                 return -EINVAL;
1015         }
1016         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1017
1018
1019 #ifdef CONFIG_SMP
1020         /* if this isn't the CPU which is the parent of the kobj, we
1021          * only need to unlink, put and exit
1022          */
1023         if (unlikely(cpu != data->cpu)) {
1024                 dprintk("removing link\n");
1025                 cpumask_clear_cpu(cpu, data->cpus);
1026                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1027                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1028                 cpufreq_cpu_put(data);
1029                 cpufreq_debug_enable_ratelimit();
1030                 unlock_policy_rwsem_write(cpu);
1031                 return 0;
1032         }
1033 #endif
1034
1035 #ifdef CONFIG_SMP
1036
1037 #ifdef CONFIG_HOTPLUG_CPU
1038         per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1039 #endif
1040
1041         /* if we have other CPUs still registered, we need to unlink them,
1042          * or else wait_for_completion below will lock up. Clean the
1043          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1044          * the sysfs links afterwards.
1045          */
1046         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1047                 for_each_cpu(j, data->cpus) {
1048                         if (j == cpu)
1049                                 continue;
1050                         per_cpu(cpufreq_cpu_data, j) = NULL;
1051                 }
1052         }
1053
1054         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1055
1056         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1057                 for_each_cpu(j, data->cpus) {
1058                         if (j == cpu)
1059                                 continue;
1060                         dprintk("removing link for cpu %u\n", j);
1061 #ifdef CONFIG_HOTPLUG_CPU
1062                         per_cpu(cpufreq_cpu_governor, j) = data->governor;
1063 #endif
1064                         cpu_sys_dev = get_cpu_sysdev(j);
1065                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1066                         cpufreq_cpu_put(data);
1067                 }
1068         }
1069 #else
1070         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1071 #endif
1072
1073         if (cpufreq_driver->target)
1074                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1075
1076         unlock_policy_rwsem_write(cpu);
1077
1078         kobject_put(&data->kobj);
1079
1080         /* we need to make sure that the underlying kobj is actually
1081          * not referenced anymore by anybody before we proceed with
1082          * unloading.
1083          */
1084         dprintk("waiting for dropping of refcount\n");
1085         wait_for_completion(&data->kobj_unregister);
1086         dprintk("wait complete\n");
1087
1088         if (cpufreq_driver->exit)
1089                 cpufreq_driver->exit(data);
1090
1091         free_cpumask_var(data->related_cpus);
1092         free_cpumask_var(data->cpus);
1093         kfree(data);
1094         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1095
1096         cpufreq_debug_enable_ratelimit();
1097         return 0;
1098 }
1099
1100
1101 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1102 {
1103         unsigned int cpu = sys_dev->id;
1104         int retval;
1105
1106         if (cpu_is_offline(cpu))
1107                 return 0;
1108
1109         if (unlikely(lock_policy_rwsem_write(cpu)))
1110                 BUG();
1111
1112         retval = __cpufreq_remove_dev(sys_dev);
1113         return retval;
1114 }
1115
1116
1117 static void handle_update(struct work_struct *work)
1118 {
1119         struct cpufreq_policy *policy =
1120                 container_of(work, struct cpufreq_policy, update);
1121         unsigned int cpu = policy->cpu;
1122         dprintk("handle_update for cpu %u called\n", cpu);
1123         cpufreq_update_policy(cpu);
1124 }
1125
1126 /**
1127  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1128  *      @cpu: cpu number
1129  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1130  *      @new_freq: CPU frequency the CPU actually runs at
1131  *
1132  *      We adjust to current frequency first, and need to clean up later.
1133  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1134  */
1135 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1136                                 unsigned int new_freq)
1137 {
1138         struct cpufreq_freqs freqs;
1139
1140         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1141                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1142
1143         freqs.cpu = cpu;
1144         freqs.old = old_freq;
1145         freqs.new = new_freq;
1146         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1147         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1148 }
1149
1150
1151 /**
1152  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1153  * @cpu: CPU number
1154  *
1155  * This is the last known freq, without actually getting it from the driver.
1156  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1157  */
1158 unsigned int cpufreq_quick_get(unsigned int cpu)
1159 {
1160         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1161         unsigned int ret_freq = 0;
1162
1163         if (policy) {
1164                 ret_freq = policy->cur;
1165                 cpufreq_cpu_put(policy);
1166         }
1167
1168         return ret_freq;
1169 }
1170 EXPORT_SYMBOL(cpufreq_quick_get);
1171
1172
1173 static unsigned int __cpufreq_get(unsigned int cpu)
1174 {
1175         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1176         unsigned int ret_freq = 0;
1177
1178         if (!cpufreq_driver->get)
1179                 return ret_freq;
1180
1181         ret_freq = cpufreq_driver->get(cpu);
1182
1183         if (ret_freq && policy->cur &&
1184                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1185                 /* verify no discrepancy between actual and
1186                                         saved value exists */
1187                 if (unlikely(ret_freq != policy->cur)) {
1188                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1189                         schedule_work(&policy->update);
1190                 }
1191         }
1192
1193         return ret_freq;
1194 }
1195
1196 /**
1197  * cpufreq_get - get the current CPU frequency (in kHz)
1198  * @cpu: CPU number
1199  *
1200  * Get the CPU current (static) CPU frequency
1201  */
1202 unsigned int cpufreq_get(unsigned int cpu)
1203 {
1204         unsigned int ret_freq = 0;
1205         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1206
1207         if (!policy)
1208                 goto out;
1209
1210         if (unlikely(lock_policy_rwsem_read(cpu)))
1211                 goto out_policy;
1212
1213         ret_freq = __cpufreq_get(cpu);
1214
1215         unlock_policy_rwsem_read(cpu);
1216
1217 out_policy:
1218         cpufreq_cpu_put(policy);
1219 out:
1220         return ret_freq;
1221 }
1222 EXPORT_SYMBOL(cpufreq_get);
1223
1224
1225 /**
1226  *      cpufreq_suspend - let the low level driver prepare for suspend
1227  */
1228
1229 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1230 {
1231         int cpu = sysdev->id;
1232         int ret = 0;
1233         unsigned int cur_freq = 0;
1234         struct cpufreq_policy *cpu_policy;
1235
1236         dprintk("suspending cpu %u\n", cpu);
1237
1238         if (!cpu_online(cpu))
1239                 return 0;
1240
1241         /* we may be lax here as interrupts are off. Nonetheless
1242          * we need to grab the correct cpu policy, as to check
1243          * whether we really run on this CPU.
1244          */
1245
1246         cpu_policy = cpufreq_cpu_get(cpu);
1247         if (!cpu_policy)
1248                 return -EINVAL;
1249
1250         /* only handle each CPU group once */
1251         if (unlikely(cpu_policy->cpu != cpu))
1252                 goto out;
1253
1254         if (cpufreq_driver->suspend) {
1255                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1256                 if (ret) {
1257                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1258                                         "step on CPU %u\n", cpu_policy->cpu);
1259                         goto out;
1260                 }
1261         }
1262
1263         if (cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)
1264                 goto out;
1265
1266         if (cpufreq_driver->get)
1267                 cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1268
1269         if (!cur_freq || !cpu_policy->cur) {
1270                 printk(KERN_ERR "cpufreq: suspend failed to assert current "
1271                        "frequency is what timing core thinks it is.\n");
1272                 goto out;
1273         }
1274
1275         if (unlikely(cur_freq != cpu_policy->cur)) {
1276                 struct cpufreq_freqs freqs;
1277
1278                 if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1279                         dprintk("Warning: CPU frequency is %u, "
1280                                "cpufreq assumed %u kHz.\n",
1281                                cur_freq, cpu_policy->cur);
1282
1283                 freqs.cpu = cpu;
1284                 freqs.old = cpu_policy->cur;
1285                 freqs.new = cur_freq;
1286
1287                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
1288                                     CPUFREQ_SUSPENDCHANGE, &freqs);
1289                 adjust_jiffies(CPUFREQ_SUSPENDCHANGE, &freqs);
1290
1291                 cpu_policy->cur = cur_freq;
1292         }
1293
1294 out:
1295         cpufreq_cpu_put(cpu_policy);
1296         return ret;
1297 }
1298
1299 /**
1300  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1301  *
1302  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1303  *      2.) if ->target and !CPUFREQ_CONST_LOOPS: verify we're in sync
1304  *      3.) schedule call cpufreq_update_policy() ASAP as interrupts are
1305  *          restored.
1306  */
1307 static int cpufreq_resume(struct sys_device *sysdev)
1308 {
1309         int cpu = sysdev->id;
1310         int ret = 0;
1311         struct cpufreq_policy *cpu_policy;
1312
1313         dprintk("resuming cpu %u\n", cpu);
1314
1315         if (!cpu_online(cpu))
1316                 return 0;
1317
1318         /* we may be lax here as interrupts are off. Nonetheless
1319          * we need to grab the correct cpu policy, as to check
1320          * whether we really run on this CPU.
1321          */
1322
1323         cpu_policy = cpufreq_cpu_get(cpu);
1324         if (!cpu_policy)
1325                 return -EINVAL;
1326
1327         /* only handle each CPU group once */
1328         if (unlikely(cpu_policy->cpu != cpu))
1329                 goto fail;
1330
1331         if (cpufreq_driver->resume) {
1332                 ret = cpufreq_driver->resume(cpu_policy);
1333                 if (ret) {
1334                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1335                                         "step on CPU %u\n", cpu_policy->cpu);
1336                         goto fail;
1337                 }
1338         }
1339
1340         if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1341                 unsigned int cur_freq = 0;
1342
1343                 if (cpufreq_driver->get)
1344                         cur_freq = cpufreq_driver->get(cpu_policy->cpu);
1345
1346                 if (!cur_freq || !cpu_policy->cur) {
1347                         printk(KERN_ERR "cpufreq: resume failed to assert "
1348                                         "current frequency is what timing core "
1349                                         "thinks it is.\n");
1350                         goto out;
1351                 }
1352
1353                 if (unlikely(cur_freq != cpu_policy->cur)) {
1354                         struct cpufreq_freqs freqs;
1355
1356                         if (!(cpufreq_driver->flags & CPUFREQ_PM_NO_WARN))
1357                                 dprintk("Warning: CPU frequency "
1358                                        "is %u, cpufreq assumed %u kHz.\n",
1359                                        cur_freq, cpu_policy->cur);
1360
1361                         freqs.cpu = cpu;
1362                         freqs.old = cpu_policy->cur;
1363                         freqs.new = cur_freq;
1364
1365                         srcu_notifier_call_chain(
1366                                         &cpufreq_transition_notifier_list,
1367                                         CPUFREQ_RESUMECHANGE, &freqs);
1368                         adjust_jiffies(CPUFREQ_RESUMECHANGE, &freqs);
1369
1370                         cpu_policy->cur = cur_freq;
1371                 }
1372         }
1373
1374 out:
1375         schedule_work(&cpu_policy->update);
1376 fail:
1377         cpufreq_cpu_put(cpu_policy);
1378         return ret;
1379 }
1380
1381 static struct sysdev_driver cpufreq_sysdev_driver = {
1382         .add            = cpufreq_add_dev,
1383         .remove         = cpufreq_remove_dev,
1384         .suspend        = cpufreq_suspend,
1385         .resume         = cpufreq_resume,
1386 };
1387
1388
1389 /*********************************************************************
1390  *                     NOTIFIER LISTS INTERFACE                      *
1391  *********************************************************************/
1392
1393 /**
1394  *      cpufreq_register_notifier - register a driver with cpufreq
1395  *      @nb: notifier function to register
1396  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1397  *
1398  *      Add a driver to one of two lists: either a list of drivers that
1399  *      are notified about clock rate changes (once before and once after
1400  *      the transition), or a list of drivers that are notified about
1401  *      changes in cpufreq policy.
1402  *
1403  *      This function may sleep, and has the same return conditions as
1404  *      blocking_notifier_chain_register.
1405  */
1406 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1407 {
1408         int ret;
1409
1410         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1411
1412         switch (list) {
1413         case CPUFREQ_TRANSITION_NOTIFIER:
1414                 ret = srcu_notifier_chain_register(
1415                                 &cpufreq_transition_notifier_list, nb);
1416                 break;
1417         case CPUFREQ_POLICY_NOTIFIER:
1418                 ret = blocking_notifier_chain_register(
1419                                 &cpufreq_policy_notifier_list, nb);
1420                 break;
1421         default:
1422                 ret = -EINVAL;
1423         }
1424
1425         return ret;
1426 }
1427 EXPORT_SYMBOL(cpufreq_register_notifier);
1428
1429
1430 /**
1431  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1432  *      @nb: notifier block to be unregistered
1433  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1434  *
1435  *      Remove a driver from the CPU frequency notifier list.
1436  *
1437  *      This function may sleep, and has the same return conditions as
1438  *      blocking_notifier_chain_unregister.
1439  */
1440 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1441 {
1442         int ret;
1443
1444         switch (list) {
1445         case CPUFREQ_TRANSITION_NOTIFIER:
1446                 ret = srcu_notifier_chain_unregister(
1447                                 &cpufreq_transition_notifier_list, nb);
1448                 break;
1449         case CPUFREQ_POLICY_NOTIFIER:
1450                 ret = blocking_notifier_chain_unregister(
1451                                 &cpufreq_policy_notifier_list, nb);
1452                 break;
1453         default:
1454                 ret = -EINVAL;
1455         }
1456
1457         return ret;
1458 }
1459 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1460
1461
1462 /*********************************************************************
1463  *                              GOVERNORS                            *
1464  *********************************************************************/
1465
1466
1467 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1468                             unsigned int target_freq,
1469                             unsigned int relation)
1470 {
1471         int retval = -EINVAL;
1472
1473         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1474                 target_freq, relation);
1475         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1476                 retval = cpufreq_driver->target(policy, target_freq, relation);
1477
1478         return retval;
1479 }
1480 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1481
1482 int cpufreq_driver_target(struct cpufreq_policy *policy,
1483                           unsigned int target_freq,
1484                           unsigned int relation)
1485 {
1486         int ret = -EINVAL;
1487
1488         policy = cpufreq_cpu_get(policy->cpu);
1489         if (!policy)
1490                 goto no_policy;
1491
1492         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1493                 goto fail;
1494
1495         ret = __cpufreq_driver_target(policy, target_freq, relation);
1496
1497         unlock_policy_rwsem_write(policy->cpu);
1498
1499 fail:
1500         cpufreq_cpu_put(policy);
1501 no_policy:
1502         return ret;
1503 }
1504 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1505
1506 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1507 {
1508         int ret = 0;
1509
1510         policy = cpufreq_cpu_get(policy->cpu);
1511         if (!policy)
1512                 return -EINVAL;
1513
1514         if (cpu_online(cpu) && cpufreq_driver->getavg)
1515                 ret = cpufreq_driver->getavg(policy, cpu);
1516
1517         cpufreq_cpu_put(policy);
1518         return ret;
1519 }
1520 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1521
1522 /*
1523  * when "event" is CPUFREQ_GOV_LIMITS
1524  */
1525
1526 static int __cpufreq_governor(struct cpufreq_policy *policy,
1527                                         unsigned int event)
1528 {
1529         int ret;
1530
1531         /* Only must be defined when default governor is known to have latency
1532            restrictions, like e.g. conservative or ondemand.
1533            That this is the case is already ensured in Kconfig
1534         */
1535 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1536         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1537 #else
1538         struct cpufreq_governor *gov = NULL;
1539 #endif
1540
1541         if (policy->governor->max_transition_latency &&
1542             policy->cpuinfo.transition_latency >
1543             policy->governor->max_transition_latency) {
1544                 if (!gov)
1545                         return -EINVAL;
1546                 else {
1547                         printk(KERN_WARNING "%s governor failed, too long"
1548                                " transition latency of HW, fallback"
1549                                " to %s governor\n",
1550                                policy->governor->name,
1551                                gov->name);
1552                         policy->governor = gov;
1553                 }
1554         }
1555
1556         if (!try_module_get(policy->governor->owner))
1557                 return -EINVAL;
1558
1559         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1560                                                 policy->cpu, event);
1561         ret = policy->governor->governor(policy, event);
1562
1563         /* we keep one module reference alive for
1564                         each CPU governed by this CPU */
1565         if ((event != CPUFREQ_GOV_START) || ret)
1566                 module_put(policy->governor->owner);
1567         if ((event == CPUFREQ_GOV_STOP) && !ret)
1568                 module_put(policy->governor->owner);
1569
1570         return ret;
1571 }
1572
1573
1574 int cpufreq_register_governor(struct cpufreq_governor *governor)
1575 {
1576         int err;
1577
1578         if (!governor)
1579                 return -EINVAL;
1580
1581         mutex_lock(&cpufreq_governor_mutex);
1582
1583         err = -EBUSY;
1584         if (__find_governor(governor->name) == NULL) {
1585                 err = 0;
1586                 list_add(&governor->governor_list, &cpufreq_governor_list);
1587         }
1588
1589         mutex_unlock(&cpufreq_governor_mutex);
1590         return err;
1591 }
1592 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1593
1594
1595 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1596 {
1597         if (!governor)
1598                 return;
1599
1600         mutex_lock(&cpufreq_governor_mutex);
1601         list_del(&governor->governor_list);
1602         mutex_unlock(&cpufreq_governor_mutex);
1603         return;
1604 }
1605 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1606
1607
1608
1609 /*********************************************************************
1610  *                          POLICY INTERFACE                         *
1611  *********************************************************************/
1612
1613 /**
1614  * cpufreq_get_policy - get the current cpufreq_policy
1615  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1616  *      is written
1617  *
1618  * Reads the current cpufreq policy.
1619  */
1620 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1621 {
1622         struct cpufreq_policy *cpu_policy;
1623         if (!policy)
1624                 return -EINVAL;
1625
1626         cpu_policy = cpufreq_cpu_get(cpu);
1627         if (!cpu_policy)
1628                 return -EINVAL;
1629
1630         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1631
1632         cpufreq_cpu_put(cpu_policy);
1633         return 0;
1634 }
1635 EXPORT_SYMBOL(cpufreq_get_policy);
1636
1637
1638 /*
1639  * data   : current policy.
1640  * policy : policy to be set.
1641  */
1642 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1643                                 struct cpufreq_policy *policy)
1644 {
1645         int ret = 0;
1646
1647         cpufreq_debug_disable_ratelimit();
1648         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1649                 policy->min, policy->max);
1650
1651         memcpy(&policy->cpuinfo, &data->cpuinfo,
1652                                 sizeof(struct cpufreq_cpuinfo));
1653
1654         if (policy->min > data->max || policy->max < data->min) {
1655                 ret = -EINVAL;
1656                 goto error_out;
1657         }
1658
1659         /* verify the cpu speed can be set within this limit */
1660         ret = cpufreq_driver->verify(policy);
1661         if (ret)
1662                 goto error_out;
1663
1664         /* adjust if necessary - all reasons */
1665         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1666                         CPUFREQ_ADJUST, policy);
1667
1668         /* adjust if necessary - hardware incompatibility*/
1669         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1670                         CPUFREQ_INCOMPATIBLE, policy);
1671
1672         /* verify the cpu speed can be set within this limit,
1673            which might be different to the first one */
1674         ret = cpufreq_driver->verify(policy);
1675         if (ret)
1676                 goto error_out;
1677
1678         /* notification of the new policy */
1679         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1680                         CPUFREQ_NOTIFY, policy);
1681
1682         data->min = policy->min;
1683         data->max = policy->max;
1684
1685         dprintk("new min and max freqs are %u - %u kHz\n",
1686                                         data->min, data->max);
1687
1688         if (cpufreq_driver->setpolicy) {
1689                 data->policy = policy->policy;
1690                 dprintk("setting range\n");
1691                 ret = cpufreq_driver->setpolicy(policy);
1692         } else {
1693                 if (policy->governor != data->governor) {
1694                         /* save old, working values */
1695                         struct cpufreq_governor *old_gov = data->governor;
1696
1697                         dprintk("governor switch\n");
1698
1699                         /* end old governor */
1700                         if (data->governor)
1701                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1702
1703                         /* start new governor */
1704                         data->governor = policy->governor;
1705                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1706                                 /* new governor failed, so re-start old one */
1707                                 dprintk("starting governor %s failed\n",
1708                                                         data->governor->name);
1709                                 if (old_gov) {
1710                                         data->governor = old_gov;
1711                                         __cpufreq_governor(data,
1712                                                            CPUFREQ_GOV_START);
1713                                 }
1714                                 ret = -EINVAL;
1715                                 goto error_out;
1716                         }
1717                         /* might be a policy change, too, so fall through */
1718                 }
1719                 dprintk("governor: change or update limits\n");
1720                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1721         }
1722
1723 error_out:
1724         cpufreq_debug_enable_ratelimit();
1725         return ret;
1726 }
1727
1728 /**
1729  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1730  *      @cpu: CPU which shall be re-evaluated
1731  *
1732  *      Usefull for policy notifiers which have different necessities
1733  *      at different times.
1734  */
1735 int cpufreq_update_policy(unsigned int cpu)
1736 {
1737         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1738         struct cpufreq_policy policy;
1739         int ret;
1740
1741         if (!data) {
1742                 ret = -ENODEV;
1743                 goto no_policy;
1744         }
1745
1746         if (unlikely(lock_policy_rwsem_write(cpu))) {
1747                 ret = -EINVAL;
1748                 goto fail;
1749         }
1750
1751         dprintk("updating policy for CPU %u\n", cpu);
1752         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1753         policy.min = data->user_policy.min;
1754         policy.max = data->user_policy.max;
1755         policy.policy = data->user_policy.policy;
1756         policy.governor = data->user_policy.governor;
1757
1758         /* BIOS might change freq behind our back
1759           -> ask driver for current freq and notify governors about a change */
1760         if (cpufreq_driver->get) {
1761                 policy.cur = cpufreq_driver->get(cpu);
1762                 if (!data->cur) {
1763                         dprintk("Driver did not initialize current freq");
1764                         data->cur = policy.cur;
1765                 } else {
1766                         if (data->cur != policy.cur)
1767                                 cpufreq_out_of_sync(cpu, data->cur,
1768                                                                 policy.cur);
1769                 }
1770         }
1771
1772         ret = __cpufreq_set_policy(data, &policy);
1773
1774         unlock_policy_rwsem_write(cpu);
1775
1776 fail:
1777         cpufreq_cpu_put(data);
1778 no_policy:
1779         return ret;
1780 }
1781 EXPORT_SYMBOL(cpufreq_update_policy);
1782
1783 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1784                                         unsigned long action, void *hcpu)
1785 {
1786         unsigned int cpu = (unsigned long)hcpu;
1787         struct sys_device *sys_dev;
1788
1789         sys_dev = get_cpu_sysdev(cpu);
1790         if (sys_dev) {
1791                 switch (action) {
1792                 case CPU_ONLINE:
1793                 case CPU_ONLINE_FROZEN:
1794                         cpufreq_add_dev(sys_dev);
1795                         break;
1796                 case CPU_DOWN_PREPARE:
1797                 case CPU_DOWN_PREPARE_FROZEN:
1798                         if (unlikely(lock_policy_rwsem_write(cpu)))
1799                                 BUG();
1800
1801                         __cpufreq_remove_dev(sys_dev);
1802                         break;
1803                 case CPU_DOWN_FAILED:
1804                 case CPU_DOWN_FAILED_FROZEN:
1805                         cpufreq_add_dev(sys_dev);
1806                         break;
1807                 }
1808         }
1809         return NOTIFY_OK;
1810 }
1811
1812 static struct notifier_block __refdata cpufreq_cpu_notifier =
1813 {
1814     .notifier_call = cpufreq_cpu_callback,
1815 };
1816
1817 /*********************************************************************
1818  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1819  *********************************************************************/
1820
1821 /**
1822  * cpufreq_register_driver - register a CPU Frequency driver
1823  * @driver_data: A struct cpufreq_driver containing the values#
1824  * submitted by the CPU Frequency driver.
1825  *
1826  *   Registers a CPU Frequency driver to this core code. This code
1827  * returns zero on success, -EBUSY when another driver got here first
1828  * (and isn't unregistered in the meantime).
1829  *
1830  */
1831 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1832 {
1833         unsigned long flags;
1834         int ret;
1835
1836         if (!driver_data || !driver_data->verify || !driver_data->init ||
1837             ((!driver_data->setpolicy) && (!driver_data->target)))
1838                 return -EINVAL;
1839
1840         dprintk("trying to register driver %s\n", driver_data->name);
1841
1842         if (driver_data->setpolicy)
1843                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1844
1845         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1846         if (cpufreq_driver) {
1847                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1848                 return -EBUSY;
1849         }
1850         cpufreq_driver = driver_data;
1851         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1852
1853         ret = sysdev_driver_register(&cpu_sysdev_class,
1854                                         &cpufreq_sysdev_driver);
1855
1856         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1857                 int i;
1858                 ret = -ENODEV;
1859
1860                 /* check for at least one working CPU */
1861                 for (i = 0; i < nr_cpu_ids; i++)
1862                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1863                                 ret = 0;
1864                                 break;
1865                         }
1866
1867                 /* if all ->init() calls failed, unregister */
1868                 if (ret) {
1869                         dprintk("no CPU initialized for driver %s\n",
1870                                                         driver_data->name);
1871                         sysdev_driver_unregister(&cpu_sysdev_class,
1872                                                 &cpufreq_sysdev_driver);
1873
1874                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1875                         cpufreq_driver = NULL;
1876                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1877                 }
1878         }
1879
1880         if (!ret) {
1881                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1882                 dprintk("driver %s up and running\n", driver_data->name);
1883                 cpufreq_debug_enable_ratelimit();
1884         }
1885
1886         return ret;
1887 }
1888 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1889
1890
1891 /**
1892  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1893  *
1894  *    Unregister the current CPUFreq driver. Only call this if you have
1895  * the right to do so, i.e. if you have succeeded in initialising before!
1896  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1897  * currently not initialised.
1898  */
1899 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1900 {
1901         unsigned long flags;
1902
1903         cpufreq_debug_disable_ratelimit();
1904
1905         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1906                 cpufreq_debug_enable_ratelimit();
1907                 return -EINVAL;
1908         }
1909
1910         dprintk("unregistering driver %s\n", driver->name);
1911
1912         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1913         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1914
1915         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1916         cpufreq_driver = NULL;
1917         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1918
1919         return 0;
1920 }
1921 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1922
1923 static int __init cpufreq_core_init(void)
1924 {
1925         int cpu;
1926
1927         for_each_possible_cpu(cpu) {
1928                 per_cpu(policy_cpu, cpu) = -1;
1929                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1930         }
1931         return 0;
1932 }
1933
1934 core_initcall(cpufreq_core_init);