drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31 #include <linux/syscore_ops.h>
  32
  33 #include <trace/events/power.h>
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  * - Lock should not be held across
  65  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  66  */
  67 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  68 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  69
  70 #define lock_policy_rwsem(mode, cpu)                                    \
  71 static int lock_policy_rwsem_##mode                                     \
  72 (int cpu)                                                               \
  73 {                                                                       \
  74         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  75         BUG_ON(policy_cpu == -1);                                       \
  76         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  77         if (unlikely(!cpu_online(cpu))) {                               \
  78                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  79                 return -1;                                              \
  80         }                                                               \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86
  87 lock_policy_rwsem(write, cpu);
  88
  89 static void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95
  96 static void unlock_policy_rwsem_write(int cpu)
  97 {
  98         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
  99         BUG_ON(policy_cpu == -1);
 100         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 101 }
 102
 103
 104 /* internal prototypes */
 105 static int __cpufreq_governor(struct cpufreq_policy *policy,
 106                 unsigned int event);
 107 static unsigned int __cpufreq_get(unsigned int cpu);
 108 static void handle_update(struct work_struct *work);
 109
 110 /**
 111  * Two notifier lists: the "policy" list is involved in the
 112  * validation process for a new CPU frequency policy; the
 113  * "transition" list for kernel code that needs to handle
 114  * changes to devices when the CPU clock speed changes.
 115  * The mutex locks both lists.
 116  */
 117 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 118 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 119
 120 static bool init_cpufreq_transition_notifier_list_called;
 121 static int __init init_cpufreq_transition_notifier_list(void)
 122 {
 123         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 124         init_cpufreq_transition_notifier_list_called = true;
 125         return 0;
 126 }
 127 pure_initcall(init_cpufreq_transition_notifier_list);
 128
 129 static int off __read_mostly;
 130 int cpufreq_disabled(void)
 131 {
 132         return off;
 133 }
 134 void disable_cpufreq(void)
 135 {
 136         off = 1;
 137 }
 138 static LIST_HEAD(cpufreq_governor_list);
 139 static DEFINE_MUTEX(cpufreq_governor_mutex);
 140
 141 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 142 {
 143         struct cpufreq_policy *data;
 144         unsigned long flags;
 145
 146         if (cpu >= nr_cpu_ids)
 147                 goto err_out;
 148
 149         /* get the cpufreq driver */
 150         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 151
 152         if (!cpufreq_driver)
 153                 goto err_out_unlock;
 154
 155         if (!try_module_get(cpufreq_driver->owner))
 156                 goto err_out_unlock;
 157
 158
 159         /* get the CPU */
 160         data = per_cpu(cpufreq_cpu_data, cpu);
 161
 162         if (!data)
 163                 goto err_out_put_module;
 164
 165         if (!kobject_get(&data->kobj))
 166                 goto err_out_put_module;
 167
 168         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 169         return data;
 170
 171 err_out_put_module:
 172         module_put(cpufreq_driver->owner);
 173 err_out_unlock:
 174         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 175 err_out:
 176         return NULL;
 177 }
 178 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 179
 180
 181 void cpufreq_cpu_put(struct cpufreq_policy *data)
 182 {
 183         kobject_put(&data->kobj);
 184         module_put(cpufreq_driver->owner);
 185 }
 186 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 187
 188
 189 /*********************************************************************
 190  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 191  *********************************************************************/
 192
 193 /**
 194  * adjust_jiffies - adjust the system "loops_per_jiffy"
 195  *
 196  * This function alters the system "loops_per_jiffy" for the clock
 197  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 198  * systems as each CPU might be scaled differently. So, use the arch
 199  * per-CPU loops_per_jiffy value wherever possible.
 200  */
 201 #ifndef CONFIG_SMP
 202 static unsigned long l_p_j_ref;
 203 static unsigned int  l_p_j_ref_freq;
 204
 205 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 206 {
 207         if (ci->flags & CPUFREQ_CONST_LOOPS)
 208                 return;
 209
 210         if (!l_p_j_ref_freq) {
 211                 l_p_j_ref = loops_per_jiffy;
 212                 l_p_j_ref_freq = ci->old;
 213                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 214                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 215         }
 216         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 217             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 218             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 219                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 220                                                                 ci->new);
 221                 pr_debug("scaling loops_per_jiffy to %lu "
 222                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 223         }
 224 }
 225 #else
 226 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 227 {
 228         return;
 229 }
 230 #endif
 231
 232
 233 /**
 234  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 235  * on frequency transition.
 236  *
 237  * This function calls the transition notifiers and the "adjust_jiffies"
 238  * function. It is called twice on all CPU frequency changes that have
 239  * external effects.
 240  */
 241 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 242 {
 243         struct cpufreq_policy *policy;
 244
 245         BUG_ON(irqs_disabled());
 246
 247         freqs->flags = cpufreq_driver->flags;
 248         pr_debug("notification %u of frequency transition to %u kHz\n",
 249                 state, freqs->new);
 250
 251         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 252         switch (state) {
 253
 254         case CPUFREQ_PRECHANGE:
 255                 /* detect if the driver reported a value as "old frequency"
 256                  * which is not equal to what the cpufreq core thinks is
 257                  * "old frequency".
 258                  */
 259                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 260                         if ((policy) && (policy->cpu == freqs->cpu) &&
 261                             (policy->cur) && (policy->cur != freqs->old)) {
 262                                 pr_debug("Warning: CPU frequency is"
 263                                         " %u, cpufreq assumed %u kHz.\n",
 264                                         freqs->old, policy->cur);
 265                                 freqs->old = policy->cur;
 266                         }
 267                 }
 268                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 269                                 CPUFREQ_PRECHANGE, freqs);
 270                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 271                 break;
 272
 273         case CPUFREQ_POSTCHANGE:
 274                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 275                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 276                         (unsigned long)freqs->cpu);
 277                 trace_power_frequency(POWER_PSTATE, freqs->new, freqs->cpu);
 278                 trace_cpu_frequency(freqs->new, freqs->cpu);
 279                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 280                                 CPUFREQ_POSTCHANGE, freqs);
 281                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 282                         policy->cur = freqs->new;
 283                 break;
 284         }
 285 }
 286 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 287
 288
 289
 290 /*********************************************************************
 291  *                          SYSFS INTERFACE                          *
 292  *********************************************************************/
 293
 294 static struct cpufreq_governor *__find_governor(const char *str_governor)
 295 {
 296         struct cpufreq_governor *t;
 297
 298         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 299                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 300                         return t;
 301
 302         return NULL;
 303 }
 304
 305 /**
 306  * cpufreq_parse_governor - parse a governor string
 307  */
 308 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 309                                 struct cpufreq_governor **governor)
 310 {
 311         int err = -EINVAL;
 312
 313         if (!cpufreq_driver)
 314                 goto out;
 315
 316         if (cpufreq_driver->setpolicy) {
 317                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 318                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 319                         err = 0;
 320                 } else if (!strnicmp(str_governor, "powersave",
 321                                                 CPUFREQ_NAME_LEN)) {
 322                         *policy = CPUFREQ_POLICY_POWERSAVE;
 323                         err = 0;
 324                 }
 325         } else if (cpufreq_driver->target) {
 326                 struct cpufreq_governor *t;
 327
 328                 mutex_lock(&cpufreq_governor_mutex);
 329
 330                 t = __find_governor(str_governor);
 331
 332                 if (t == NULL) {
 333                         int ret;
 334
 335                         mutex_unlock(&cpufreq_governor_mutex);
 336                         ret = request_module("cpufreq_%s", str_governor);
 337                         mutex_lock(&cpufreq_governor_mutex);
 338
 339                         if (ret == 0)
 340                                 t = __find_governor(str_governor);
 341                 }
 342
 343                 if (t != NULL) {
 344                         *governor = t;
 345                         err = 0;
 346                 }
 347
 348                 mutex_unlock(&cpufreq_governor_mutex);
 349         }
 350 out:
 351         return err;
 352 }
 353
 354
 355 /**
 356  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 357  * print out cpufreq information
 358  *
 359  * Write out information from cpufreq_driver->policy[cpu]; object must be
 360  * "unsigned int".
 361  */
 362
 363 #define show_one(file_name, object)                     \
 364 static ssize_t show_##file_name                         \
 365 (struct cpufreq_policy *policy, char *buf)              \
 366 {                                                       \
 367         return sprintf(buf, "%u\n", policy->object);    \
 368 }
 369
 370 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 371 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 372 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 373 show_one(scaling_min_freq, min);
 374 show_one(scaling_max_freq, max);
 375 show_one(scaling_cur_freq, cur);
 376
 377 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 378                                 struct cpufreq_policy *policy);
 379
 380 /**
 381  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 382  */
 383 #define store_one(file_name, object)                    \
 384 static ssize_t store_##file_name                                        \
 385 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 386 {                                                                       \
 387         unsigned int ret = -EINVAL;                                     \
 388         struct cpufreq_policy new_policy;                               \
 389                                                                         \
 390         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 391         if (ret)                                                        \
 392                 return -EINVAL;                                         \
 393                                                                         \
 394         ret = sscanf(buf, "%u", &new_policy.object);                    \
 395         if (ret != 1)                                                   \
 396                 return -EINVAL;                                         \
 397                                                                         \
 398         ret = __cpufreq_set_policy(policy, &new_policy);                \
 399         policy->user_policy.object = policy->object;                    \
 400                                                                         \
 401         return ret ? ret : count;                                       \
 402 }
 403
 404 store_one(scaling_min_freq, min);
 405 store_one(scaling_max_freq, max);
 406
 407 /**
 408  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 409  */
 410 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 411                                         char *buf)
 412 {
 413         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 414         if (!cur_freq)
 415                 return sprintf(buf, "<unknown>");
 416         return sprintf(buf, "%u\n", cur_freq);
 417 }
 418
 419
 420 /**
 421  * show_scaling_governor - show the current policy for the specified CPU
 422  */
 423 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 424 {
 425         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 426                 return sprintf(buf, "powersave\n");
 427         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 428                 return sprintf(buf, "performance\n");
 429         else if (policy->governor)
 430                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 431                                 policy->governor->name);
 432         return -EINVAL;
 433 }
 434
 435
 436 /**
 437  * store_scaling_governor - store policy for the specified CPU
 438  */
 439 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 440                                         const char *buf, size_t count)
 441 {
 442         unsigned int ret = -EINVAL;
 443         char    str_governor[16];
 444         struct cpufreq_policy new_policy;
 445
 446         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 447         if (ret)
 448                 return ret;
 449
 450         ret = sscanf(buf, "%15s", str_governor);
 451         if (ret != 1)
 452                 return -EINVAL;
 453
 454         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 455                                                 &new_policy.governor))
 456                 return -EINVAL;
 457
 458         /* Do not use cpufreq_set_policy here or the user_policy.max
 459            will be wrongly overridden */
 460         ret = __cpufreq_set_policy(policy, &new_policy);
 461
 462         policy->user_policy.policy = policy->policy;
 463         policy->user_policy.governor = policy->governor;
 464
 465         if (ret)
 466                 return ret;
 467         else
 468                 return count;
 469 }
 470
 471 /**
 472  * show_scaling_driver - show the cpufreq driver currently loaded
 473  */
 474 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 475 {
 476         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 477 }
 478
 479 /**
 480  * show_scaling_available_governors - show the available CPUfreq governors
 481  */
 482 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 483                                                 char *buf)
 484 {
 485         ssize_t i = 0;
 486         struct cpufreq_governor *t;
 487
 488         if (!cpufreq_driver->target) {
 489                 i += sprintf(buf, "performance powersave");
 490                 goto out;
 491         }
 492
 493         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 494                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 495                     - (CPUFREQ_NAME_LEN + 2)))
 496                         goto out;
 497                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 498         }
 499 out:
 500         i += sprintf(&buf[i], "\n");
 501         return i;
 502 }
 503
 504 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 505 {
 506         ssize_t i = 0;
 507         unsigned int cpu;
 508
 509         for_each_cpu(cpu, mask) {
 510                 if (i)
 511                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 512                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 513                 if (i >= (PAGE_SIZE - 5))
 514                         break;
 515         }
 516         i += sprintf(&buf[i], "\n");
 517         return i;
 518 }
 519
 520 /**
 521  * show_related_cpus - show the CPUs affected by each transition even if
 522  * hw coordination is in use
 523  */
 524 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 525 {
 526         if (cpumask_empty(policy->related_cpus))
 527                 return show_cpus(policy->cpus, buf);
 528         return show_cpus(policy->related_cpus, buf);
 529 }
 530
 531 /**
 532  * show_affected_cpus - show the CPUs affected by each transition
 533  */
 534 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 535 {
 536         return show_cpus(policy->cpus, buf);
 537 }
 538
 539 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 540                                         const char *buf, size_t count)
 541 {
 542         unsigned int freq = 0;
 543         unsigned int ret;
 544
 545         if (!policy->governor || !policy->governor->store_setspeed)
 546                 return -EINVAL;
 547
 548         ret = sscanf(buf, "%u", &freq);
 549         if (ret != 1)
 550                 return -EINVAL;
 551
 552         policy->governor->store_setspeed(policy, freq);
 553
 554         return count;
 555 }
 556
 557 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 558 {
 559         if (!policy->governor || !policy->governor->show_setspeed)
 560                 return sprintf(buf, "<unsupported>\n");
 561
 562         return policy->governor->show_setspeed(policy, buf);
 563 }
 564
 565 /**
 566  * show_scaling_driver - show the current cpufreq HW/BIOS limitation
 567  */
 568 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 569 {
 570         unsigned int limit;
 571         int ret;
 572         if (cpufreq_driver->bios_limit) {
 573                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 574                 if (!ret)
 575                         return sprintf(buf, "%u\n", limit);
 576         }
 577         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 578 }
 579
 580 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 581 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 582 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 583 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 584 cpufreq_freq_attr_ro(scaling_available_governors);
 585 cpufreq_freq_attr_ro(scaling_driver);
 586 cpufreq_freq_attr_ro(scaling_cur_freq);
 587 cpufreq_freq_attr_ro(bios_limit);
 588 cpufreq_freq_attr_ro(related_cpus);
 589 cpufreq_freq_attr_ro(affected_cpus);
 590 cpufreq_freq_attr_rw(scaling_min_freq);
 591 cpufreq_freq_attr_rw(scaling_max_freq);
 592 cpufreq_freq_attr_rw(scaling_governor);
 593 cpufreq_freq_attr_rw(scaling_setspeed);
 594
 595 static struct attribute *default_attrs[] = {
 596         &cpuinfo_min_freq.attr,
 597         &cpuinfo_max_freq.attr,
 598         &cpuinfo_transition_latency.attr,
 599         &scaling_min_freq.attr,
 600         &scaling_max_freq.attr,
 601         &affected_cpus.attr,
 602         &related_cpus.attr,
 603         &scaling_governor.attr,
 604         &scaling_driver.attr,
 605         &scaling_available_governors.attr,
 606         &scaling_setspeed.attr,
 607         NULL
 608 };
 609
 610 struct kobject *cpufreq_global_kobject;
 611 EXPORT_SYMBOL(cpufreq_global_kobject);
 612
 613 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 614 #define to_attr(a) container_of(a, struct freq_attr, attr)
 615
 616 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 617 {
 618         struct cpufreq_policy *policy = to_policy(kobj);
 619         struct freq_attr *fattr = to_attr(attr);
 620         ssize_t ret = -EINVAL;
 621         policy = cpufreq_cpu_get(policy->cpu);
 622         if (!policy)
 623                 goto no_policy;
 624
 625         if (lock_policy_rwsem_read(policy->cpu) < 0)
 626                 goto fail;
 627
 628         if (fattr->show)
 629                 ret = fattr->show(policy, buf);
 630         else
 631                 ret = -EIO;
 632
 633         unlock_policy_rwsem_read(policy->cpu);
 634 fail:
 635         cpufreq_cpu_put(policy);
 636 no_policy:
 637         return ret;
 638 }
 639
 640 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 641                      const char *buf, size_t count)
 642 {
 643         struct cpufreq_policy *policy = to_policy(kobj);
 644         struct freq_attr *fattr = to_attr(attr);
 645         ssize_t ret = -EINVAL;
 646         policy = cpufreq_cpu_get(policy->cpu);
 647         if (!policy)
 648                 goto no_policy;
 649
 650         if (lock_policy_rwsem_write(policy->cpu) < 0)
 651                 goto fail;
 652
 653         if (fattr->store)
 654                 ret = fattr->store(policy, buf, count);
 655         else
 656                 ret = -EIO;
 657
 658         unlock_policy_rwsem_write(policy->cpu);
 659 fail:
 660         cpufreq_cpu_put(policy);
 661 no_policy:
 662         return ret;
 663 }
 664
 665 static void cpufreq_sysfs_release(struct kobject *kobj)
 666 {
 667         struct cpufreq_policy *policy = to_policy(kobj);
 668         pr_debug("last reference is dropped\n");
 669         complete(&policy->kobj_unregister);
 670 }
 671
 672 static const struct sysfs_ops sysfs_ops = {
 673         .show   = show,
 674         .store  = store,
 675 };
 676
 677 static struct kobj_type ktype_cpufreq = {
 678         .sysfs_ops      = &sysfs_ops,
 679         .default_attrs  = default_attrs,
 680         .release        = cpufreq_sysfs_release,
 681 };
 682
 683 /*
 684  * Returns:
 685  *   Negative: Failure
 686  *   0:        Success
 687  *   Positive: When we have a managed CPU and the sysfs got symlinked
 688  */
 689 static int cpufreq_add_dev_policy(unsigned int cpu,
 690                                   struct cpufreq_policy *policy,
 691                                   struct device *dev)
 692 {
 693         int ret = 0;
 694 #ifdef CONFIG_SMP
 695         unsigned long flags;
 696         unsigned int j;
 697 #ifdef CONFIG_HOTPLUG_CPU
 698         struct cpufreq_governor *gov;
 699
 700         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
 701         if (gov) {
 702                 policy->governor = gov;
 703                 pr_debug("Restoring governor %s for cpu %d\n",
 704                        policy->governor->name, cpu);
 705         }
 706 #endif
 707
 708         for_each_cpu(j, policy->cpus) {
 709                 struct cpufreq_policy *managed_policy;
 710
 711                 if (cpu == j)
 712                         continue;
 713
 714                 /* Check for existing affected CPUs.
 715                  * They may not be aware of it due to CPU Hotplug.
 716                  * cpufreq_cpu_put is called when the device is removed
 717                  * in __cpufreq_remove_dev()
 718                  */
 719                 managed_policy = cpufreq_cpu_get(j);
 720                 if (unlikely(managed_policy)) {
 721
 722                         /* Set proper policy_cpu */
 723                         unlock_policy_rwsem_write(cpu);
 724                         per_cpu(cpufreq_policy_cpu, cpu) = managed_policy->cpu;
 725
 726                         if (lock_policy_rwsem_write(cpu) < 0) {
 727                                 /* Should not go through policy unlock path */
 728                                 if (cpufreq_driver->exit)
 729                                         cpufreq_driver->exit(policy);
 730                                 cpufreq_cpu_put(managed_policy);
 731                                 return -EBUSY;
 732                         }
 733
 734                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 735                         cpumask_copy(managed_policy->cpus, policy->cpus);
 736                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 737                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 738
 739                         pr_debug("CPU already managed, adding link\n");
 740                         ret = sysfs_create_link(&dev->kobj,
 741                                                 &managed_policy->kobj,
 742                                                 "cpufreq");
 743                         if (ret)
 744                                 cpufreq_cpu_put(managed_policy);
 745                         /*
 746                          * Success. We only needed to be added to the mask.
 747                          * Call driver->exit() because only the cpu parent of
 748                          * the kobj needed to call init().
 749                          */
 750                         if (cpufreq_driver->exit)
 751                                 cpufreq_driver->exit(policy);
 752
 753                         if (!ret)
 754                                 return 1;
 755                         else
 756                                 return ret;
 757                 }
 758         }
 759 #endif
 760         return ret;
 761 }
 762
 763
 764 /* symlink affected CPUs */
 765 static int cpufreq_add_dev_symlink(unsigned int cpu,
 766                                    struct cpufreq_policy *policy)
 767 {
 768         unsigned int j;
 769         int ret = 0;
 770
 771         for_each_cpu(j, policy->cpus) {
 772                 struct cpufreq_policy *managed_policy;
 773                 struct device *cpu_dev;
 774
 775                 if (j == cpu)
 776                         continue;
 777                 if (!cpu_online(j))
 778                         continue;
 779
 780                 pr_debug("CPU %u already managed, adding link\n", j);
 781                 managed_policy = cpufreq_cpu_get(cpu);
 782                 cpu_dev = get_cpu_device(j);
 783                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 784                                         "cpufreq");
 785                 if (ret) {
 786                         cpufreq_cpu_put(managed_policy);
 787                         return ret;
 788                 }
 789         }
 790         return ret;
 791 }
 792
 793 static int cpufreq_add_dev_interface(unsigned int cpu,
 794                                      struct cpufreq_policy *policy,
 795                                      struct device *dev)
 796 {
 797         struct cpufreq_policy new_policy;
 798         struct freq_attr **drv_attr;
 799         unsigned long flags;
 800         int ret = 0;
 801         unsigned int j;
 802
 803         /* prepare interface data */
 804         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 805                                    &dev->kobj, "cpufreq");
 806         if (ret)
 807                 return ret;
 808
 809         /* set up files for this cpu device */
 810         drv_attr = cpufreq_driver->attr;
 811         while ((drv_attr) && (*drv_attr)) {
 812                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 813                 if (ret)
 814                         goto err_out_kobj_put;
 815                 drv_attr++;
 816         }
 817         if (cpufreq_driver->get) {
 818                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 819                 if (ret)
 820                         goto err_out_kobj_put;
 821         }
 822         if (cpufreq_driver->target) {
 823                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 824                 if (ret)
 825                         goto err_out_kobj_put;
 826         }
 827         if (cpufreq_driver->bios_limit) {
 828                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 829                 if (ret)
 830                         goto err_out_kobj_put;
 831         }
 832
 833         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 834         for_each_cpu(j, policy->cpus) {
 835                 if (!cpu_online(j))
 836                         continue;
 837                 per_cpu(cpufreq_cpu_data, j) = policy;
 838                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 839         }
 840         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 841
 842         ret = cpufreq_add_dev_symlink(cpu, policy);
 843         if (ret)
 844                 goto err_out_kobj_put;
 845
 846         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 847         /* assure that the starting sequence is run in __cpufreq_set_policy */
 848         policy->governor = NULL;
 849
 850         /* set default policy */
 851         ret = __cpufreq_set_policy(policy, &new_policy);
 852         policy->user_policy.policy = policy->policy;
 853         policy->user_policy.governor = policy->governor;
 854
 855         if (ret) {
 856                 pr_debug("setting policy failed\n");
 857                 if (cpufreq_driver->exit)
 858                         cpufreq_driver->exit(policy);
 859         }
 860         return ret;
 861
 862 err_out_kobj_put:
 863         kobject_put(&policy->kobj);
 864         wait_for_completion(&policy->kobj_unregister);
 865         return ret;
 866 }
 867
 868
 869 /**
 870  * cpufreq_add_dev - add a CPU device
 871  *
 872  * Adds the cpufreq interface for a CPU device.
 873  *
 874  * The Oracle says: try running cpufreq registration/unregistration concurrently
 875  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 876  * mess up, but more thorough testing is needed. - Mathieu
 877  */
 878 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 879 {
 880         unsigned int cpu = dev->id;
 881         int ret = 0, found = 0;
 882         struct cpufreq_policy *policy;
 883         unsigned long flags;
 884         unsigned int j;
 885 #ifdef CONFIG_HOTPLUG_CPU
 886         int sibling;
 887 #endif
 888
 889         if (cpu_is_offline(cpu))
 890                 return 0;
 891
 892         pr_debug("adding CPU %u\n", cpu);
 893
 894 #ifdef CONFIG_SMP
 895         /* check whether a different CPU already registered this
 896          * CPU because it is in the same boat. */
 897         policy = cpufreq_cpu_get(cpu);
 898         if (unlikely(policy)) {
 899                 cpufreq_cpu_put(policy);
 900                 return 0;
 901         }
 902 #endif
 903
 904         if (!try_module_get(cpufreq_driver->owner)) {
 905                 ret = -EINVAL;
 906                 goto module_out;
 907         }
 908
 909         ret = -ENOMEM;
 910         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 911         if (!policy)
 912                 goto nomem_out;
 913
 914         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 915                 goto err_free_policy;
 916
 917         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 918                 goto err_free_cpumask;
 919
 920         policy->cpu = cpu;
 921         cpumask_copy(policy->cpus, cpumask_of(cpu));
 922
 923         /* Initially set CPU itself as the policy_cpu */
 924         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
 925         ret = (lock_policy_rwsem_write(cpu) < 0);
 926         WARN_ON(ret);
 927
 928         init_completion(&policy->kobj_unregister);
 929         INIT_WORK(&policy->update, handle_update);
 930
 931         /* Set governor before ->init, so that driver could check it */
 932 #ifdef CONFIG_HOTPLUG_CPU
 933         for_each_online_cpu(sibling) {
 934                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 935                 if (cp && cp->governor &&
 936                     (cpumask_test_cpu(cpu, cp->related_cpus))) {
 937                         policy->governor = cp->governor;
 938                         found = 1;
 939                         break;
 940                 }
 941         }
 942 #endif
 943         if (!found)
 944                 policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 945         /* call driver. From then on the cpufreq must be able
 946          * to accept all calls to ->verify and ->setpolicy for this CPU
 947          */
 948         ret = cpufreq_driver->init(policy);
 949         if (ret) {
 950                 pr_debug("initialization failed\n");
 951                 goto err_unlock_policy;
 952         }
 953         policy->user_policy.min = policy->min;
 954         policy->user_policy.max = policy->max;
 955
 956         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 957                                      CPUFREQ_START, policy);
 958
 959         ret = cpufreq_add_dev_policy(cpu, policy, dev);
 960         if (ret) {
 961                 if (ret > 0)
 962                         /* This is a managed cpu, symlink created,
 963                            exit with 0 */
 964                         ret = 0;
 965                 goto err_unlock_policy;
 966         }
 967
 968         ret = cpufreq_add_dev_interface(cpu, policy, dev);
 969         if (ret)
 970                 goto err_out_unregister;
 971
 972         unlock_policy_rwsem_write(cpu);
 973
 974         kobject_uevent(&policy->kobj, KOBJ_ADD);
 975         module_put(cpufreq_driver->owner);
 976         pr_debug("initialization complete\n");
 977
 978         return 0;
 979
 980
 981 err_out_unregister:
 982         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 983         for_each_cpu(j, policy->cpus)
 984                 per_cpu(cpufreq_cpu_data, j) = NULL;
 985         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 986
 987         kobject_put(&policy->kobj);
 988         wait_for_completion(&policy->kobj_unregister);
 989
 990 err_unlock_policy:
 991         unlock_policy_rwsem_write(cpu);
 992         free_cpumask_var(policy->related_cpus);
 993 err_free_cpumask:
 994         free_cpumask_var(policy->cpus);
 995 err_free_policy:
 996         kfree(policy);
 997 nomem_out:
 998         module_put(cpufreq_driver->owner);
 999 module_out:
1000         return ret;
1001 }
1002
1003
1004 /**
1005  * __cpufreq_remove_dev - remove a CPU device
1006  *
1007  * Removes the cpufreq interface for a CPU device.
1008  * Caller should already have policy_rwsem in write mode for this CPU.
1009  * This routine frees the rwsem before returning.
1010  */
1011 static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1012 {
1013         unsigned int cpu = dev->id;
1014         unsigned long flags;
1015         struct cpufreq_policy *data;
1016         struct kobject *kobj;
1017         struct completion *cmp;
1018 #ifdef CONFIG_SMP
1019         struct device *cpu_dev;
1020         unsigned int j;
1021 #endif
1022
1023         pr_debug("unregistering CPU %u\n", cpu);
1024
1025         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1026         data = per_cpu(cpufreq_cpu_data, cpu);
1027
1028         if (!data) {
1029                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1030                 unlock_policy_rwsem_write(cpu);
1031                 return -EINVAL;
1032         }
1033         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1034
1035
1036 #ifdef CONFIG_SMP
1037         /* if this isn't the CPU which is the parent of the kobj, we
1038          * only need to unlink, put and exit
1039          */
1040         if (unlikely(cpu != data->cpu)) {
1041                 pr_debug("removing link\n");
1042                 cpumask_clear_cpu(cpu, data->cpus);
1043                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1044                 kobj = &dev->kobj;
1045                 cpufreq_cpu_put(data);
1046                 unlock_policy_rwsem_write(cpu);
1047                 sysfs_remove_link(kobj, "cpufreq");
1048                 return 0;
1049         }
1050 #endif
1051
1052 #ifdef CONFIG_SMP
1053
1054 #ifdef CONFIG_HOTPLUG_CPU
1055         strncpy(per_cpu(cpufreq_cpu_governor, cpu), data->governor->name,
1056                         CPUFREQ_NAME_LEN);
1057 #endif
1058
1059         /* if we have other CPUs still registered, we need to unlink them,
1060          * or else wait_for_completion below will lock up. Clean the
1061          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1062          * the sysfs links afterwards.
1063          */
1064         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1065                 for_each_cpu(j, data->cpus) {
1066                         if (j == cpu)
1067                                 continue;
1068                         per_cpu(cpufreq_cpu_data, j) = NULL;
1069                 }
1070         }
1071
1072         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1073
1074         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1075                 for_each_cpu(j, data->cpus) {
1076                         if (j == cpu)
1077                                 continue;
1078                         pr_debug("removing link for cpu %u\n", j);
1079 #ifdef CONFIG_HOTPLUG_CPU
1080                         strncpy(per_cpu(cpufreq_cpu_governor, j),
1081                                 data->governor->name, CPUFREQ_NAME_LEN);
1082 #endif
1083                         cpu_dev = get_cpu_device(j);
1084                         kobj = &cpu_dev->kobj;
1085                         unlock_policy_rwsem_write(cpu);
1086                         sysfs_remove_link(kobj, "cpufreq");
1087                         lock_policy_rwsem_write(cpu);
1088                         cpufreq_cpu_put(data);
1089                 }
1090         }
1091 #else
1092         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1093 #endif
1094
1095         if (cpufreq_driver->target)
1096                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1097
1098         kobj = &data->kobj;
1099         cmp = &data->kobj_unregister;
1100         unlock_policy_rwsem_write(cpu);
1101         kobject_put(kobj);
1102
1103         /* we need to make sure that the underlying kobj is actually
1104          * not referenced anymore by anybody before we proceed with
1105          * unloading.
1106          */
1107         pr_debug("waiting for dropping of refcount\n");
1108         wait_for_completion(cmp);
1109         pr_debug("wait complete\n");
1110
1111         lock_policy_rwsem_write(cpu);
1112         if (cpufreq_driver->exit)
1113                 cpufreq_driver->exit(data);
1114         unlock_policy_rwsem_write(cpu);
1115
1116 #ifdef CONFIG_HOTPLUG_CPU
1117         /* when the CPU which is the parent of the kobj is hotplugged
1118          * offline, check for siblings, and create cpufreq sysfs interface
1119          * and symlinks
1120          */
1121         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1122                 /* first sibling now owns the new sysfs dir */
1123                 cpumask_clear_cpu(cpu, data->cpus);
1124                 cpufreq_add_dev(get_cpu_device(cpumask_first(data->cpus)), NULL);
1125
1126                 /* finally remove our own symlink */
1127                 lock_policy_rwsem_write(cpu);
1128                 __cpufreq_remove_dev(dev, sif);
1129         }
1130 #endif
1131
1132         free_cpumask_var(data->related_cpus);
1133         free_cpumask_var(data->cpus);
1134         kfree(data);
1135
1136         return 0;
1137 }
1138
1139
1140 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1141 {
1142         unsigned int cpu = dev->id;
1143         int retval;
1144
1145         if (cpu_is_offline(cpu))
1146                 return 0;
1147
1148         if (unlikely(lock_policy_rwsem_write(cpu)))
1149                 BUG();
1150
1151         retval = __cpufreq_remove_dev(dev, sif);
1152         return retval;
1153 }
1154
1155
1156 static void handle_update(struct work_struct *work)
1157 {
1158         struct cpufreq_policy *policy =
1159                 container_of(work, struct cpufreq_policy, update);
1160         unsigned int cpu = policy->cpu;
1161         pr_debug("handle_update for cpu %u called\n", cpu);
1162         cpufreq_update_policy(cpu);
1163 }
1164
1165 /**
1166  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1167  *      @cpu: cpu number
1168  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1169  *      @new_freq: CPU frequency the CPU actually runs at
1170  *
1171  *      We adjust to current frequency first, and need to clean up later.
1172  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1173  */
1174 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1175                                 unsigned int new_freq)
1176 {
1177         struct cpufreq_freqs freqs;
1178
1179         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1180                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1181
1182         freqs.cpu = cpu;
1183         freqs.old = old_freq;
1184         freqs.new = new_freq;
1185         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1186         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1187 }
1188
1189
1190 /**
1191  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1192  * @cpu: CPU number
1193  *
1194  * This is the last known freq, without actually getting it from the driver.
1195  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1196  */
1197 unsigned int cpufreq_quick_get(unsigned int cpu)
1198 {
1199         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1200         unsigned int ret_freq = 0;
1201
1202         if (policy) {
1203                 ret_freq = policy->cur;
1204                 cpufreq_cpu_put(policy);
1205         }
1206
1207         return ret_freq;
1208 }
1209 EXPORT_SYMBOL(cpufreq_quick_get);
1210
1211 /**
1212  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1213  * @cpu: CPU number
1214  *
1215  * Just return the max possible frequency for a given CPU.
1216  */
1217 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1218 {
1219         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1220         unsigned int ret_freq = 0;
1221
1222         if (policy) {
1223                 ret_freq = policy->max;
1224                 cpufreq_cpu_put(policy);
1225         }
1226
1227         return ret_freq;
1228 }
1229 EXPORT_SYMBOL(cpufreq_quick_get_max);
1230
1231
1232 static unsigned int __cpufreq_get(unsigned int cpu)
1233 {
1234         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1235         unsigned int ret_freq = 0;
1236
1237         if (!cpufreq_driver->get)
1238                 return ret_freq;
1239
1240         ret_freq = cpufreq_driver->get(cpu);
1241
1242         if (ret_freq && policy->cur &&
1243                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1244                 /* verify no discrepancy between actual and
1245                                         saved value exists */
1246                 if (unlikely(ret_freq != policy->cur)) {
1247                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1248                         schedule_work(&policy->update);
1249                 }
1250         }
1251
1252         return ret_freq;
1253 }
1254
1255 /**
1256  * cpufreq_get - get the current CPU frequency (in kHz)
1257  * @cpu: CPU number
1258  *
1259  * Get the CPU current (static) CPU frequency
1260  */
1261 unsigned int cpufreq_get(unsigned int cpu)
1262 {
1263         unsigned int ret_freq = 0;
1264         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1265
1266         if (!policy)
1267                 goto out;
1268
1269         if (unlikely(lock_policy_rwsem_read(cpu)))
1270                 goto out_policy;
1271
1272         ret_freq = __cpufreq_get(cpu);
1273
1274         unlock_policy_rwsem_read(cpu);
1275
1276 out_policy:
1277         cpufreq_cpu_put(policy);
1278 out:
1279         return ret_freq;
1280 }
1281 EXPORT_SYMBOL(cpufreq_get);
1282
1283 static struct subsys_interface cpufreq_interface = {
1284         .name           = "cpufreq",
1285         .subsys         = &cpu_subsys,
1286         .add_dev        = cpufreq_add_dev,
1287         .remove_dev     = cpufreq_remove_dev,
1288 };
1289
1290
1291 /**
1292  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1293  *
1294  * This function is only executed for the boot processor.  The other CPUs
1295  * have been put offline by means of CPU hotplug.
1296  */
1297 static int cpufreq_bp_suspend(void)
1298 {
1299         int ret = 0;
1300
1301         int cpu = smp_processor_id();
1302         struct cpufreq_policy *cpu_policy;
1303
1304         pr_debug("suspending cpu %u\n", cpu);
1305
1306         /* If there's no policy for the boot CPU, we have nothing to do. */
1307         cpu_policy = cpufreq_cpu_get(cpu);
1308         if (!cpu_policy)
1309                 return 0;
1310
1311         if (cpufreq_driver->suspend) {
1312                 ret = cpufreq_driver->suspend(cpu_policy);
1313                 if (ret)
1314                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1315                                         "step on CPU %u\n", cpu_policy->cpu);
1316         }
1317
1318         cpufreq_cpu_put(cpu_policy);
1319         return ret;
1320 }
1321
1322 /**
1323  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1324  *
1325  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1326  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1327  *          restored. It will verify that the current freq is in sync with
1328  *          what we believe it to be. This is a bit later than when it
1329  *          should be, but nonethteless it's better than calling
1330  *          cpufreq_driver->get() here which might re-enable interrupts...
1331  *
1332  * This function is only executed for the boot CPU.  The other CPUs have not
1333  * been turned on yet.
1334  */
1335 static void cpufreq_bp_resume(void)
1336 {
1337         int ret = 0;
1338
1339         int cpu = smp_processor_id();
1340         struct cpufreq_policy *cpu_policy;
1341
1342         pr_debug("resuming cpu %u\n", cpu);
1343
1344         /* If there's no policy for the boot CPU, we have nothing to do. */
1345         cpu_policy = cpufreq_cpu_get(cpu);
1346         if (!cpu_policy)
1347                 return;
1348
1349         if (cpufreq_driver->resume) {
1350                 ret = cpufreq_driver->resume(cpu_policy);
1351                 if (ret) {
1352                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1353                                         "step on CPU %u\n", cpu_policy->cpu);
1354                         goto fail;
1355                 }
1356         }
1357
1358         schedule_work(&cpu_policy->update);
1359
1360 fail:
1361         cpufreq_cpu_put(cpu_policy);
1362 }
1363
1364 static struct syscore_ops cpufreq_syscore_ops = {
1365         .suspend        = cpufreq_bp_suspend,
1366         .resume         = cpufreq_bp_resume,
1367 };
1368
1369
1370 /*********************************************************************
1371  *                     NOTIFIER LISTS INTERFACE                      *
1372  *********************************************************************/
1373
1374 /**
1375  *      cpufreq_register_notifier - register a driver with cpufreq
1376  *      @nb: notifier function to register
1377  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1378  *
1379  *      Add a driver to one of two lists: either a list of drivers that
1380  *      are notified about clock rate changes (once before and once after
1381  *      the transition), or a list of drivers that are notified about
1382  *      changes in cpufreq policy.
1383  *
1384  *      This function may sleep, and has the same return conditions as
1385  *      blocking_notifier_chain_register.
1386  */
1387 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1388 {
1389         int ret;
1390
1391         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1392
1393         switch (list) {
1394         case CPUFREQ_TRANSITION_NOTIFIER:
1395                 ret = srcu_notifier_chain_register(
1396                                 &cpufreq_transition_notifier_list, nb);
1397                 break;
1398         case CPUFREQ_POLICY_NOTIFIER:
1399                 ret = blocking_notifier_chain_register(
1400                                 &cpufreq_policy_notifier_list, nb);
1401                 break;
1402         default:
1403                 ret = -EINVAL;
1404         }
1405
1406         return ret;
1407 }
1408 EXPORT_SYMBOL(cpufreq_register_notifier);
1409
1410
1411 /**
1412  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1413  *      @nb: notifier block to be unregistered
1414  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1415  *
1416  *      Remove a driver from the CPU frequency notifier list.
1417  *
1418  *      This function may sleep, and has the same return conditions as
1419  *      blocking_notifier_chain_unregister.
1420  */
1421 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1422 {
1423         int ret;
1424
1425         switch (list) {
1426         case CPUFREQ_TRANSITION_NOTIFIER:
1427                 ret = srcu_notifier_chain_unregister(
1428                                 &cpufreq_transition_notifier_list, nb);
1429                 break;
1430         case CPUFREQ_POLICY_NOTIFIER:
1431                 ret = blocking_notifier_chain_unregister(
1432                                 &cpufreq_policy_notifier_list, nb);
1433                 break;
1434         default:
1435                 ret = -EINVAL;
1436         }
1437
1438         return ret;
1439 }
1440 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1441
1442
1443 /*********************************************************************
1444  *                              GOVERNORS                            *
1445  *********************************************************************/
1446
1447
1448 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1449                             unsigned int target_freq,
1450                             unsigned int relation)
1451 {
1452         int retval = -EINVAL;
1453
1454         if (cpufreq_disabled())
1455                 return -ENODEV;
1456
1457         pr_debug("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1458                 target_freq, relation);
1459         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1460                 retval = cpufreq_driver->target(policy, target_freq, relation);
1461
1462         return retval;
1463 }
1464 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1465
1466 int cpufreq_driver_target(struct cpufreq_policy *policy,
1467                           unsigned int target_freq,
1468                           unsigned int relation)
1469 {
1470         int ret = -EINVAL;
1471
1472         policy = cpufreq_cpu_get(policy->cpu);
1473         if (!policy)
1474                 goto no_policy;
1475
1476         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1477                 goto fail;
1478
1479         ret = __cpufreq_driver_target(policy, target_freq, relation);
1480
1481         unlock_policy_rwsem_write(policy->cpu);
1482
1483 fail:
1484         cpufreq_cpu_put(policy);
1485 no_policy:
1486         return ret;
1487 }
1488 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1489
1490 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1491 {
1492         int ret = 0;
1493
1494         policy = cpufreq_cpu_get(policy->cpu);
1495         if (!policy)
1496                 return -EINVAL;
1497
1498         if (cpu_online(cpu) && cpufreq_driver->getavg)
1499                 ret = cpufreq_driver->getavg(policy, cpu);
1500
1501         cpufreq_cpu_put(policy);
1502         return ret;
1503 }
1504 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1505
1506 /*
1507  * when "event" is CPUFREQ_GOV_LIMITS
1508  */
1509
1510 static int __cpufreq_governor(struct cpufreq_policy *policy,
1511                                         unsigned int event)
1512 {
1513         int ret;
1514
1515         /* Only must be defined when default governor is known to have latency
1516            restrictions, like e.g. conservative or ondemand.
1517            That this is the case is already ensured in Kconfig
1518         */
1519 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1520         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1521 #else
1522         struct cpufreq_governor *gov = NULL;
1523 #endif
1524
1525         if (policy->governor->max_transition_latency &&
1526             policy->cpuinfo.transition_latency >
1527             policy->governor->max_transition_latency) {
1528                 if (!gov)
1529                         return -EINVAL;
1530                 else {
1531                         printk(KERN_WARNING "%s governor failed, too long"
1532                                " transition latency of HW, fallback"
1533                                " to %s governor\n",
1534                                policy->governor->name,
1535                                gov->name);
1536                         policy->governor = gov;
1537                 }
1538         }
1539
1540         if (!try_module_get(policy->governor->owner))
1541                 return -EINVAL;
1542
1543         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1544                                                 policy->cpu, event);
1545         ret = policy->governor->governor(policy, event);
1546
1547         /* we keep one module reference alive for
1548                         each CPU governed by this CPU */
1549         if ((event != CPUFREQ_GOV_START) || ret)
1550                 module_put(policy->governor->owner);
1551         if ((event == CPUFREQ_GOV_STOP) && !ret)
1552                 module_put(policy->governor->owner);
1553
1554         return ret;
1555 }
1556
1557
1558 int cpufreq_register_governor(struct cpufreq_governor *governor)
1559 {
1560         int err;
1561
1562         if (!governor)
1563                 return -EINVAL;
1564
1565         if (cpufreq_disabled())
1566                 return -ENODEV;
1567
1568         mutex_lock(&cpufreq_governor_mutex);
1569
1570         err = -EBUSY;
1571         if (__find_governor(governor->name) == NULL) {
1572                 err = 0;
1573                 list_add(&governor->governor_list, &cpufreq_governor_list);
1574         }
1575
1576         mutex_unlock(&cpufreq_governor_mutex);
1577         return err;
1578 }
1579 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1580
1581
1582 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1583 {
1584 #ifdef CONFIG_HOTPLUG_CPU
1585         int cpu;
1586 #endif
1587
1588         if (!governor)
1589                 return;
1590
1591         if (cpufreq_disabled())
1592                 return;
1593
1594 #ifdef CONFIG_HOTPLUG_CPU
1595         for_each_present_cpu(cpu) {
1596                 if (cpu_online(cpu))
1597                         continue;
1598                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1599                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1600         }
1601 #endif
1602
1603         mutex_lock(&cpufreq_governor_mutex);
1604         list_del(&governor->governor_list);
1605         mutex_unlock(&cpufreq_governor_mutex);
1606         return;
1607 }
1608 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1609
1610
1611
1612 /*********************************************************************
1613  *                          POLICY INTERFACE                         *
1614  *********************************************************************/
1615
1616 /**
1617  * cpufreq_get_policy - get the current cpufreq_policy
1618  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1619  *      is written
1620  *
1621  * Reads the current cpufreq policy.
1622  */
1623 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1624 {
1625         struct cpufreq_policy *cpu_policy;
1626         if (!policy)
1627                 return -EINVAL;
1628
1629         cpu_policy = cpufreq_cpu_get(cpu);
1630         if (!cpu_policy)
1631                 return -EINVAL;
1632
1633         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1634
1635         cpufreq_cpu_put(cpu_policy);
1636         return 0;
1637 }
1638 EXPORT_SYMBOL(cpufreq_get_policy);
1639
1640
1641 /*
1642  * data   : current policy.
1643  * policy : policy to be set.
1644  */
1645 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1646                                 struct cpufreq_policy *policy)
1647 {
1648         int ret = 0;
1649
1650         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1651                 policy->min, policy->max);
1652
1653         memcpy(&policy->cpuinfo, &data->cpuinfo,
1654                                 sizeof(struct cpufreq_cpuinfo));
1655
1656         if (policy->min > data->max || policy->max < data->min) {
1657                 ret = -EINVAL;
1658                 goto error_out;
1659         }
1660
1661         /* verify the cpu speed can be set within this limit */
1662         ret = cpufreq_driver->verify(policy);
1663         if (ret)
1664                 goto error_out;
1665
1666         /* adjust if necessary - all reasons */
1667         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1668                         CPUFREQ_ADJUST, policy);
1669
1670         /* adjust if necessary - hardware incompatibility*/
1671         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1672                         CPUFREQ_INCOMPATIBLE, policy);
1673
1674         /* verify the cpu speed can be set within this limit,
1675            which might be different to the first one */
1676         ret = cpufreq_driver->verify(policy);
1677         if (ret)
1678                 goto error_out;
1679
1680         /* notification of the new policy */
1681         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1682                         CPUFREQ_NOTIFY, policy);
1683
1684         data->min = policy->min;
1685         data->max = policy->max;
1686
1687         pr_debug("new min and max freqs are %u - %u kHz\n",
1688                                         data->min, data->max);
1689
1690         if (cpufreq_driver->setpolicy) {
1691                 data->policy = policy->policy;
1692                 pr_debug("setting range\n");
1693                 ret = cpufreq_driver->setpolicy(policy);
1694         } else {
1695                 if (policy->governor != data->governor) {
1696                         /* save old, working values */
1697                         struct cpufreq_governor *old_gov = data->governor;
1698
1699                         pr_debug("governor switch\n");
1700
1701                         /* end old governor */
1702                         if (data->governor)
1703                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1704
1705                         /* start new governor */
1706                         data->governor = policy->governor;
1707                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1708                                 /* new governor failed, so re-start old one */
1709                                 pr_debug("starting governor %s failed\n",
1710                                                         data->governor->name);
1711                                 if (old_gov) {
1712                                         data->governor = old_gov;
1713                                         __cpufreq_governor(data,
1714                                                            CPUFREQ_GOV_START);
1715                                 }
1716                                 ret = -EINVAL;
1717                                 goto error_out;
1718                         }
1719                         /* might be a policy change, too, so fall through */
1720                 }
1721                 pr_debug("governor: change or update limits\n");
1722                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1723         }
1724
1725 error_out:
1726         return ret;
1727 }
1728
1729 /**
1730  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1731  *      @cpu: CPU which shall be re-evaluated
1732  *
1733  *      Useful for policy notifiers which have different necessities
1734  *      at different times.
1735  */
1736 int cpufreq_update_policy(unsigned int cpu)
1737 {
1738         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1739         struct cpufreq_policy policy;
1740         int ret;
1741
1742         if (!data) {
1743                 ret = -ENODEV;
1744                 goto no_policy;
1745         }
1746
1747         if (unlikely(lock_policy_rwsem_write(cpu))) {
1748                 ret = -EINVAL;
1749                 goto fail;
1750         }
1751
1752         pr_debug("updating policy for CPU %u\n", cpu);
1753         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1754         policy.min = data->user_policy.min;
1755         policy.max = data->user_policy.max;
1756         policy.policy = data->user_policy.policy;
1757         policy.governor = data->user_policy.governor;
1758
1759         /* BIOS might change freq behind our back
1760           -> ask driver for current freq and notify governors about a change */
1761         if (cpufreq_driver->get) {
1762                 policy.cur = cpufreq_driver->get(cpu);
1763                 if (!data->cur) {
1764                         pr_debug("Driver did not initialize current freq");
1765                         data->cur = policy.cur;
1766                 } else {
1767                         if (data->cur != policy.cur)
1768                                 cpufreq_out_of_sync(cpu, data->cur,
1769                                                                 policy.cur);
1770                 }
1771         }
1772
1773         ret = __cpufreq_set_policy(data, &policy);
1774
1775         unlock_policy_rwsem_write(cpu);
1776
1777 fail:
1778         cpufreq_cpu_put(data);
1779 no_policy:
1780         return ret;
1781 }
1782 EXPORT_SYMBOL(cpufreq_update_policy);
1783
1784 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1785                                         unsigned long action, void *hcpu)
1786 {
1787         unsigned int cpu = (unsigned long)hcpu;
1788         struct device *dev;
1789
1790         dev = get_cpu_device(cpu);
1791         if (dev) {
1792                 switch (action) {
1793                 case CPU_ONLINE:
1794                 case CPU_ONLINE_FROZEN:
1795                         cpufreq_add_dev(dev, NULL);
1796                         break;
1797                 case CPU_DOWN_PREPARE:
1798                 case CPU_DOWN_PREPARE_FROZEN:
1799                         if (unlikely(lock_policy_rwsem_write(cpu)))
1800                                 BUG();
1801
1802                         __cpufreq_remove_dev(dev, NULL);
1803                         break;
1804                 case CPU_DOWN_FAILED:
1805                 case CPU_DOWN_FAILED_FROZEN:
1806                         cpufreq_add_dev(dev, NULL);
1807                         break;
1808                 }
1809         }
1810         return NOTIFY_OK;
1811 }
1812
1813 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1814     .notifier_call = cpufreq_cpu_callback,
1815 };
1816
1817 /*********************************************************************
1818  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1819  *********************************************************************/
1820
1821 /**
1822  * cpufreq_register_driver - register a CPU Frequency driver
1823  * @driver_data: A struct cpufreq_driver containing the values#
1824  * submitted by the CPU Frequency driver.
1825  *
1826  *   Registers a CPU Frequency driver to this core code. This code
1827  * returns zero on success, -EBUSY when another driver got here first
1828  * (and isn't unregistered in the meantime).
1829  *
1830  */
1831 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1832 {
1833         unsigned long flags;
1834         int ret;
1835
1836         if (cpufreq_disabled())
1837                 return -ENODEV;
1838
1839         if (!driver_data || !driver_data->verify || !driver_data->init ||
1840             ((!driver_data->setpolicy) && (!driver_data->target)))
1841                 return -EINVAL;
1842
1843         pr_debug("trying to register driver %s\n", driver_data->name);
1844
1845         if (driver_data->setpolicy)
1846                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1847
1848         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1849         if (cpufreq_driver) {
1850                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1851                 return -EBUSY;
1852         }
1853         cpufreq_driver = driver_data;
1854         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1855
1856         ret = subsys_interface_register(&cpufreq_interface);
1857         if (ret)
1858                 goto err_null_driver;
1859
1860         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1861                 int i;
1862                 ret = -ENODEV;
1863
1864                 /* check for at least one working CPU */
1865                 for (i = 0; i < nr_cpu_ids; i++)
1866                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1867                                 ret = 0;
1868                                 break;
1869                         }
1870
1871                 /* if all ->init() calls failed, unregister */
1872                 if (ret) {
1873                         pr_debug("no CPU initialized for driver %s\n",
1874                                                         driver_data->name);
1875                         goto err_if_unreg;
1876                 }
1877         }
1878
1879         register_hotcpu_notifier(&cpufreq_cpu_notifier);
1880         pr_debug("driver %s up and running\n", driver_data->name);
1881
1882         return 0;
1883 err_if_unreg:
1884         subsys_interface_unregister(&cpufreq_interface);
1885 err_null_driver:
1886         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1887         cpufreq_driver = NULL;
1888         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1889         return ret;
1890 }
1891 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1892
1893
1894 /**
1895  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1896  *
1897  *    Unregister the current CPUFreq driver. Only call this if you have
1898  * the right to do so, i.e. if you have succeeded in initialising before!
1899  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1900  * currently not initialised.
1901  */
1902 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1903 {
1904         unsigned long flags;
1905
1906         if (!cpufreq_driver || (driver != cpufreq_driver))
1907                 return -EINVAL;
1908
1909         pr_debug("unregistering driver %s\n", driver->name);
1910
1911         subsys_interface_unregister(&cpufreq_interface);
1912         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1913
1914         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1915         cpufreq_driver = NULL;
1916         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1917
1918         return 0;
1919 }
1920 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1921
1922 static int __init cpufreq_core_init(void)
1923 {
1924         int cpu;
1925
1926         if (cpufreq_disabled())
1927                 return -ENODEV;
1928
1929         for_each_possible_cpu(cpu) {
1930                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
1931                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1932         }
1933
1934         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
1935         BUG_ON(!cpufreq_global_kobject);
1936         register_syscore_ops(&cpufreq_syscore_ops);
1937
1938         return 0;
1939 }
1940 core_initcall(cpufreq_core_init);