drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/syscore_ops.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 static DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 #ifdef CONFIG_HOTPLUG_CPU
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48 #endif
  49
  50 /*
  51  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  52  * all cpufreq/hotplug/workqueue/etc related lock issues.
  53  *
  54  * The rules for this semaphore:
  55  * - Any routine that wants to read from the policy structure will
  56  *   do a down_read on this semaphore.
  57  * - Any routine that will write to the policy structure and/or may take away
  58  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  59  *   mode before doing so.
  60  *
  61  * Additional rules:
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  * - Lock should not be held across
  65  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  66  */
  67 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  68
  69 #define lock_policy_rwsem(mode, cpu)                                    \
  70 static int lock_policy_rwsem_##mode(int cpu)                            \
  71 {                                                                       \
  72         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); \
  73         BUG_ON(!policy);                                                \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy->cpu));           \
  75                                                                         \
  76         return 0;                                                       \
  77 }
  78
  79 lock_policy_rwsem(read, cpu);
  80 lock_policy_rwsem(write, cpu);
  81
  82 #define unlock_policy_rwsem(mode, cpu)                                  \
  83 static void unlock_policy_rwsem_##mode(int cpu)                         \
  84 {                                                                       \
  85         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); \
  86         BUG_ON(!policy);                                                \
  87         up_##mode(&per_cpu(cpu_policy_rwsem, policy->cpu));             \
  88 }
  89
  90 unlock_policy_rwsem(read, cpu);
  91 unlock_policy_rwsem(write, cpu);
  92
  93 /*
  94  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  95  * sections
  96  */
  97 static DECLARE_RWSEM(cpufreq_rwsem);
  98
  99 /* internal prototypes */
 100 static int __cpufreq_governor(struct cpufreq_policy *policy,
 101                 unsigned int event);
 102 static unsigned int __cpufreq_get(unsigned int cpu);
 103 static void handle_update(struct work_struct *work);
 104
 105 /**
 106  * Two notifier lists: the "policy" list is involved in the
 107  * validation process for a new CPU frequency policy; the
 108  * "transition" list for kernel code that needs to handle
 109  * changes to devices when the CPU clock speed changes.
 110  * The mutex locks both lists.
 111  */
 112 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 113 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 114
 115 static bool init_cpufreq_transition_notifier_list_called;
 116 static int __init init_cpufreq_transition_notifier_list(void)
 117 {
 118         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 119         init_cpufreq_transition_notifier_list_called = true;
 120         return 0;
 121 }
 122 pure_initcall(init_cpufreq_transition_notifier_list);
 123
 124 static int off __read_mostly;
 125 static int cpufreq_disabled(void)
 126 {
 127         return off;
 128 }
 129 void disable_cpufreq(void)
 130 {
 131         off = 1;
 132 }
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 bool have_governor_per_policy(void)
 137 {
 138         return cpufreq_driver->have_governor_per_policy;
 139 }
 140 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 141
 142 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 143 {
 144         if (have_governor_per_policy())
 145                 return &policy->kobj;
 146         else
 147                 return cpufreq_global_kobject;
 148 }
 149 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 150
 151 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 152 {
 153         u64 idle_time;
 154         u64 cur_wall_time;
 155         u64 busy_time;
 156
 157         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 158
 159         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 165
 166         idle_time = cur_wall_time - busy_time;
 167         if (wall)
 168                 *wall = cputime_to_usecs(cur_wall_time);
 169
 170         return cputime_to_usecs(idle_time);
 171 }
 172
 173 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 174 {
 175         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 176
 177         if (idle_time == -1ULL)
 178                 return get_cpu_idle_time_jiffy(cpu, wall);
 179         else if (!io_busy)
 180                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 181
 182         return idle_time;
 183 }
 184 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 185
 186 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 187 {
 188         struct cpufreq_policy *policy = NULL;
 189         unsigned long flags;
 190
 191         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 192                 return NULL;
 193
 194         if (!down_read_trylock(&cpufreq_rwsem))
 195                 return NULL;
 196
 197         /* get the cpufreq driver */
 198         read_lock_irqsave(&cpufreq_driver_lock, flags);
 199
 200         if (cpufreq_driver) {
 201                 /* get the CPU */
 202                 policy = per_cpu(cpufreq_cpu_data, cpu);
 203                 if (policy)
 204                         kobject_get(&policy->kobj);
 205         }
 206
 207         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 208
 209         if (!policy)
 210                 up_read(&cpufreq_rwsem);
 211
 212         return policy;
 213 }
 214 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 215
 216 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 217 {
 218         if (cpufreq_disabled())
 219                 return;
 220
 221         kobject_put(&policy->kobj);
 222         up_read(&cpufreq_rwsem);
 223 }
 224 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 225
 226 /*********************************************************************
 227  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 228  *********************************************************************/
 229
 230 /**
 231  * adjust_jiffies - adjust the system "loops_per_jiffy"
 232  *
 233  * This function alters the system "loops_per_jiffy" for the clock
 234  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 235  * systems as each CPU might be scaled differently. So, use the arch
 236  * per-CPU loops_per_jiffy value wherever possible.
 237  */
 238 #ifndef CONFIG_SMP
 239 static unsigned long l_p_j_ref;
 240 static unsigned int l_p_j_ref_freq;
 241
 242 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 243 {
 244         if (ci->flags & CPUFREQ_CONST_LOOPS)
 245                 return;
 246
 247         if (!l_p_j_ref_freq) {
 248                 l_p_j_ref = loops_per_jiffy;
 249                 l_p_j_ref_freq = ci->old;
 250                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 251                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 252         }
 253         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 254             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 255                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 256                                                                 ci->new);
 257                 pr_debug("scaling loops_per_jiffy to %lu "
 258                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 259         }
 260 }
 261 #else
 262 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 263 {
 264         return;
 265 }
 266 #endif
 267
 268 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 269                 struct cpufreq_freqs *freqs, unsigned int state)
 270 {
 271         BUG_ON(irqs_disabled());
 272
 273         if (cpufreq_disabled())
 274                 return;
 275
 276         freqs->flags = cpufreq_driver->flags;
 277         pr_debug("notification %u of frequency transition to %u kHz\n",
 278                 state, freqs->new);
 279
 280         switch (state) {
 281
 282         case CPUFREQ_PRECHANGE:
 283                 /* detect if the driver reported a value as "old frequency"
 284                  * which is not equal to what the cpufreq core thinks is
 285                  * "old frequency".
 286                  */
 287                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 288                         if ((policy) && (policy->cpu == freqs->cpu) &&
 289                             (policy->cur) && (policy->cur != freqs->old)) {
 290                                 pr_debug("Warning: CPU frequency is"
 291                                         " %u, cpufreq assumed %u kHz.\n",
 292                                         freqs->old, policy->cur);
 293                                 freqs->old = policy->cur;
 294                         }
 295                 }
 296                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 297                                 CPUFREQ_PRECHANGE, freqs);
 298                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 299                 break;
 300
 301         case CPUFREQ_POSTCHANGE:
 302                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 303                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 304                         (unsigned long)freqs->cpu);
 305                 trace_cpu_frequency(freqs->new, freqs->cpu);
 306                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 307                                 CPUFREQ_POSTCHANGE, freqs);
 308                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 309                         policy->cur = freqs->new;
 310                 break;
 311         }
 312 }
 313
 314 /**
 315  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 316  * on frequency transition.
 317  *
 318  * This function calls the transition notifiers and the "adjust_jiffies"
 319  * function. It is called twice on all CPU frequency changes that have
 320  * external effects.
 321  */
 322 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 323                 struct cpufreq_freqs *freqs, unsigned int state)
 324 {
 325         for_each_cpu(freqs->cpu, policy->cpus)
 326                 __cpufreq_notify_transition(policy, freqs, state);
 327 }
 328 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 329
 330
 331 /*********************************************************************
 332  *                          SYSFS INTERFACE                          *
 333  *********************************************************************/
 334
 335 static struct cpufreq_governor *__find_governor(const char *str_governor)
 336 {
 337         struct cpufreq_governor *t;
 338
 339         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 340                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 341                         return t;
 342
 343         return NULL;
 344 }
 345
 346 /**
 347  * cpufreq_parse_governor - parse a governor string
 348  */
 349 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 350                                 struct cpufreq_governor **governor)
 351 {
 352         int err = -EINVAL;
 353
 354         if (!cpufreq_driver)
 355                 goto out;
 356
 357         if (cpufreq_driver->setpolicy) {
 358                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 359                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 360                         err = 0;
 361                 } else if (!strnicmp(str_governor, "powersave",
 362                                                 CPUFREQ_NAME_LEN)) {
 363                         *policy = CPUFREQ_POLICY_POWERSAVE;
 364                         err = 0;
 365                 }
 366         } else if (cpufreq_driver->target) {
 367                 struct cpufreq_governor *t;
 368
 369                 mutex_lock(&cpufreq_governor_mutex);
 370
 371                 t = __find_governor(str_governor);
 372
 373                 if (t == NULL) {
 374                         int ret;
 375
 376                         mutex_unlock(&cpufreq_governor_mutex);
 377                         ret = request_module("cpufreq_%s", str_governor);
 378                         mutex_lock(&cpufreq_governor_mutex);
 379
 380                         if (ret == 0)
 381                                 t = __find_governor(str_governor);
 382                 }
 383
 384                 if (t != NULL) {
 385                         *governor = t;
 386                         err = 0;
 387                 }
 388
 389                 mutex_unlock(&cpufreq_governor_mutex);
 390         }
 391 out:
 392         return err;
 393 }
 394
 395 /**
 396  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 397  * print out cpufreq information
 398  *
 399  * Write out information from cpufreq_driver->policy[cpu]; object must be
 400  * "unsigned int".
 401  */
 402
 403 #define show_one(file_name, object)                     \
 404 static ssize_t show_##file_name                         \
 405 (struct cpufreq_policy *policy, char *buf)              \
 406 {                                                       \
 407         return sprintf(buf, "%u\n", policy->object);    \
 408 }
 409
 410 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 411 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 412 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 413 show_one(scaling_min_freq, min);
 414 show_one(scaling_max_freq, max);
 415 show_one(scaling_cur_freq, cur);
 416
 417 static int __cpufreq_set_policy(struct cpufreq_policy *policy,
 418                                 struct cpufreq_policy *new_policy);
 419
 420 /**
 421  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 422  */
 423 #define store_one(file_name, object)                    \
 424 static ssize_t store_##file_name                                        \
 425 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 426 {                                                                       \
 427         int ret;                                                        \
 428         struct cpufreq_policy new_policy;                               \
 429                                                                         \
 430         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 431         if (ret)                                                        \
 432                 return -EINVAL;                                         \
 433                                                                         \
 434         ret = sscanf(buf, "%u", &new_policy.object);                    \
 435         if (ret != 1)                                                   \
 436                 return -EINVAL;                                         \
 437                                                                         \
 438         ret = __cpufreq_set_policy(policy, &new_policy);                \
 439         policy->user_policy.object = policy->object;                    \
 440                                                                         \
 441         return ret ? ret : count;                                       \
 442 }
 443
 444 store_one(scaling_min_freq, min);
 445 store_one(scaling_max_freq, max);
 446
 447 /**
 448  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 449  */
 450 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 451                                         char *buf)
 452 {
 453         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 454         if (!cur_freq)
 455                 return sprintf(buf, "<unknown>");
 456         return sprintf(buf, "%u\n", cur_freq);
 457 }
 458
 459 /**
 460  * show_scaling_governor - show the current policy for the specified CPU
 461  */
 462 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 463 {
 464         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 465                 return sprintf(buf, "powersave\n");
 466         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 467                 return sprintf(buf, "performance\n");
 468         else if (policy->governor)
 469                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 470                                 policy->governor->name);
 471         return -EINVAL;
 472 }
 473
 474 /**
 475  * store_scaling_governor - store policy for the specified CPU
 476  */
 477 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 478                                         const char *buf, size_t count)
 479 {
 480         int ret;
 481         char    str_governor[16];
 482         struct cpufreq_policy new_policy;
 483
 484         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 485         if (ret)
 486                 return ret;
 487
 488         ret = sscanf(buf, "%15s", str_governor);
 489         if (ret != 1)
 490                 return -EINVAL;
 491
 492         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 493                                                 &new_policy.governor))
 494                 return -EINVAL;
 495
 496         /*
 497          * Do not use cpufreq_set_policy here or the user_policy.max
 498          * will be wrongly overridden
 499          */
 500         ret = __cpufreq_set_policy(policy, &new_policy);
 501
 502         policy->user_policy.policy = policy->policy;
 503         policy->user_policy.governor = policy->governor;
 504
 505         if (ret)
 506                 return ret;
 507         else
 508                 return count;
 509 }
 510
 511 /**
 512  * show_scaling_driver - show the cpufreq driver currently loaded
 513  */
 514 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 515 {
 516         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 517 }
 518
 519 /**
 520  * show_scaling_available_governors - show the available CPUfreq governors
 521  */
 522 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 523                                                 char *buf)
 524 {
 525         ssize_t i = 0;
 526         struct cpufreq_governor *t;
 527
 528         if (!cpufreq_driver->target) {
 529                 i += sprintf(buf, "performance powersave");
 530                 goto out;
 531         }
 532
 533         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 534                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 535                     - (CPUFREQ_NAME_LEN + 2)))
 536                         goto out;
 537                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 538         }
 539 out:
 540         i += sprintf(&buf[i], "\n");
 541         return i;
 542 }
 543
 544 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 545 {
 546         ssize_t i = 0;
 547         unsigned int cpu;
 548
 549         for_each_cpu(cpu, mask) {
 550                 if (i)
 551                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 552                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 553                 if (i >= (PAGE_SIZE - 5))
 554                         break;
 555         }
 556         i += sprintf(&buf[i], "\n");
 557         return i;
 558 }
 559 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 560
 561 /**
 562  * show_related_cpus - show the CPUs affected by each transition even if
 563  * hw coordination is in use
 564  */
 565 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 566 {
 567         return cpufreq_show_cpus(policy->related_cpus, buf);
 568 }
 569
 570 /**
 571  * show_affected_cpus - show the CPUs affected by each transition
 572  */
 573 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 574 {
 575         return cpufreq_show_cpus(policy->cpus, buf);
 576 }
 577
 578 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 579                                         const char *buf, size_t count)
 580 {
 581         unsigned int freq = 0;
 582         unsigned int ret;
 583
 584         if (!policy->governor || !policy->governor->store_setspeed)
 585                 return -EINVAL;
 586
 587         ret = sscanf(buf, "%u", &freq);
 588         if (ret != 1)
 589                 return -EINVAL;
 590
 591         policy->governor->store_setspeed(policy, freq);
 592
 593         return count;
 594 }
 595
 596 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 597 {
 598         if (!policy->governor || !policy->governor->show_setspeed)
 599                 return sprintf(buf, "<unsupported>\n");
 600
 601         return policy->governor->show_setspeed(policy, buf);
 602 }
 603
 604 /**
 605  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 606  */
 607 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 608 {
 609         unsigned int limit;
 610         int ret;
 611         if (cpufreq_driver->bios_limit) {
 612                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 613                 if (!ret)
 614                         return sprintf(buf, "%u\n", limit);
 615         }
 616         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 617 }
 618
 619 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 620 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 621 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 622 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 623 cpufreq_freq_attr_ro(scaling_available_governors);
 624 cpufreq_freq_attr_ro(scaling_driver);
 625 cpufreq_freq_attr_ro(scaling_cur_freq);
 626 cpufreq_freq_attr_ro(bios_limit);
 627 cpufreq_freq_attr_ro(related_cpus);
 628 cpufreq_freq_attr_ro(affected_cpus);
 629 cpufreq_freq_attr_rw(scaling_min_freq);
 630 cpufreq_freq_attr_rw(scaling_max_freq);
 631 cpufreq_freq_attr_rw(scaling_governor);
 632 cpufreq_freq_attr_rw(scaling_setspeed);
 633
 634 static struct attribute *default_attrs[] = {
 635         &cpuinfo_min_freq.attr,
 636         &cpuinfo_max_freq.attr,
 637         &cpuinfo_transition_latency.attr,
 638         &scaling_min_freq.attr,
 639         &scaling_max_freq.attr,
 640         &affected_cpus.attr,
 641         &related_cpus.attr,
 642         &scaling_governor.attr,
 643         &scaling_driver.attr,
 644         &scaling_available_governors.attr,
 645         &scaling_setspeed.attr,
 646         NULL
 647 };
 648
 649 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 650 #define to_attr(a) container_of(a, struct freq_attr, attr)
 651
 652 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 653 {
 654         struct cpufreq_policy *policy = to_policy(kobj);
 655         struct freq_attr *fattr = to_attr(attr);
 656         ssize_t ret = -EINVAL;
 657
 658         if (!down_read_trylock(&cpufreq_rwsem))
 659                 goto exit;
 660
 661         if (lock_policy_rwsem_read(policy->cpu) < 0)
 662                 goto up_read;
 663
 664         if (fattr->show)
 665                 ret = fattr->show(policy, buf);
 666         else
 667                 ret = -EIO;
 668
 669         unlock_policy_rwsem_read(policy->cpu);
 670
 671 up_read:
 672         up_read(&cpufreq_rwsem);
 673 exit:
 674         return ret;
 675 }
 676
 677 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 678                      const char *buf, size_t count)
 679 {
 680         struct cpufreq_policy *policy = to_policy(kobj);
 681         struct freq_attr *fattr = to_attr(attr);
 682         ssize_t ret = -EINVAL;
 683
 684         get_online_cpus();
 685
 686         if (!cpu_online(policy->cpu))
 687                 goto unlock;
 688
 689         if (!down_read_trylock(&cpufreq_rwsem))
 690                 goto unlock;
 691
 692         if (lock_policy_rwsem_write(policy->cpu) < 0)
 693                 goto up_read;
 694
 695         if (fattr->store)
 696                 ret = fattr->store(policy, buf, count);
 697         else
 698                 ret = -EIO;
 699
 700         unlock_policy_rwsem_write(policy->cpu);
 701
 702 up_read:
 703         up_read(&cpufreq_rwsem);
 704 unlock:
 705         put_online_cpus();
 706
 707         return ret;
 708 }
 709
 710 static void cpufreq_sysfs_release(struct kobject *kobj)
 711 {
 712         struct cpufreq_policy *policy = to_policy(kobj);
 713         pr_debug("last reference is dropped\n");
 714         complete(&policy->kobj_unregister);
 715 }
 716
 717 static const struct sysfs_ops sysfs_ops = {
 718         .show   = show,
 719         .store  = store,
 720 };
 721
 722 static struct kobj_type ktype_cpufreq = {
 723         .sysfs_ops      = &sysfs_ops,
 724         .default_attrs  = default_attrs,
 725         .release        = cpufreq_sysfs_release,
 726 };
 727
 728 struct kobject *cpufreq_global_kobject;
 729 EXPORT_SYMBOL(cpufreq_global_kobject);
 730
 731 static int cpufreq_global_kobject_usage;
 732
 733 int cpufreq_get_global_kobject(void)
 734 {
 735         if (!cpufreq_global_kobject_usage++)
 736                 return kobject_add(cpufreq_global_kobject,
 737                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 738
 739         return 0;
 740 }
 741 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 742
 743 void cpufreq_put_global_kobject(void)
 744 {
 745         if (!--cpufreq_global_kobject_usage)
 746                 kobject_del(cpufreq_global_kobject);
 747 }
 748 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 749
 750 int cpufreq_sysfs_create_file(const struct attribute *attr)
 751 {
 752         int ret = cpufreq_get_global_kobject();
 753
 754         if (!ret) {
 755                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 756                 if (ret)
 757                         cpufreq_put_global_kobject();
 758         }
 759
 760         return ret;
 761 }
 762 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 763
 764 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 765 {
 766         sysfs_remove_file(cpufreq_global_kobject, attr);
 767         cpufreq_put_global_kobject();
 768 }
 769 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 770
 771 /* symlink affected CPUs */
 772 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 773 {
 774         unsigned int j;
 775         int ret = 0;
 776
 777         for_each_cpu(j, policy->cpus) {
 778                 struct device *cpu_dev;
 779
 780                 if (j == policy->cpu)
 781                         continue;
 782
 783                 pr_debug("Adding link for CPU: %u\n", j);
 784                 cpu_dev = get_cpu_device(j);
 785                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 786                                         "cpufreq");
 787                 if (ret)
 788                         break;
 789         }
 790         return ret;
 791 }
 792
 793 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 794                                      struct device *dev)
 795 {
 796         struct freq_attr **drv_attr;
 797         int ret = 0;
 798
 799         /* prepare interface data */
 800         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 801                                    &dev->kobj, "cpufreq");
 802         if (ret)
 803                 return ret;
 804
 805         /* set up files for this cpu device */
 806         drv_attr = cpufreq_driver->attr;
 807         while ((drv_attr) && (*drv_attr)) {
 808                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 809                 if (ret)
 810                         goto err_out_kobj_put;
 811                 drv_attr++;
 812         }
 813         if (cpufreq_driver->get) {
 814                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 815                 if (ret)
 816                         goto err_out_kobj_put;
 817         }
 818         if (cpufreq_driver->target) {
 819                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 820                 if (ret)
 821                         goto err_out_kobj_put;
 822         }
 823         if (cpufreq_driver->bios_limit) {
 824                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 825                 if (ret)
 826                         goto err_out_kobj_put;
 827         }
 828
 829         ret = cpufreq_add_dev_symlink(policy);
 830         if (ret)
 831                 goto err_out_kobj_put;
 832
 833         return ret;
 834
 835 err_out_kobj_put:
 836         kobject_put(&policy->kobj);
 837         wait_for_completion(&policy->kobj_unregister);
 838         return ret;
 839 }
 840
 841 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 842 {
 843         struct cpufreq_policy new_policy;
 844         int ret = 0;
 845
 846         memcpy(&new_policy, policy, sizeof(*policy));
 847         /* assure that the starting sequence is run in __cpufreq_set_policy */
 848         policy->governor = NULL;
 849
 850         /* set default policy */
 851         ret = __cpufreq_set_policy(policy, &new_policy);
 852         policy->user_policy.policy = policy->policy;
 853         policy->user_policy.governor = policy->governor;
 854
 855         if (ret) {
 856                 pr_debug("setting policy failed\n");
 857                 if (cpufreq_driver->exit)
 858                         cpufreq_driver->exit(policy);
 859         }
 860 }
 861
 862 #ifdef CONFIG_HOTPLUG_CPU
 863 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 864                                   unsigned int cpu, struct device *dev,
 865                                   bool frozen)
 866 {
 867         int ret = 0, has_target = !!cpufreq_driver->target;
 868         unsigned long flags;
 869
 870         if (has_target) {
 871                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 872                 if (ret) {
 873                         pr_err("%s: Failed to stop governor\n", __func__);
 874                         return ret;
 875                 }
 876         }
 877
 878         lock_policy_rwsem_write(policy->cpu);
 879
 880         write_lock_irqsave(&cpufreq_driver_lock, flags);
 881
 882         cpumask_set_cpu(cpu, policy->cpus);
 883         per_cpu(cpufreq_cpu_data, cpu) = policy;
 884         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 885
 886         unlock_policy_rwsem_write(policy->cpu);
 887
 888         if (has_target) {
 889                 if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
 890                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
 891                         pr_err("%s: Failed to start governor\n", __func__);
 892                         return ret;
 893                 }
 894         }
 895
 896         /* Don't touch sysfs links during light-weight init */
 897         if (!frozen)
 898                 ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 899
 900         return ret;
 901 }
 902 #endif
 903
 904 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 905 {
 906         struct cpufreq_policy *policy;
 907         unsigned long flags;
 908
 909         read_lock_irqsave(&cpufreq_driver_lock, flags);
 910
 911         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 912
 913         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 914
 915         return policy;
 916 }
 917
 918 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 919 {
 920         struct cpufreq_policy *policy;
 921
 922         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 923         if (!policy)
 924                 return NULL;
 925
 926         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 927                 goto err_free_policy;
 928
 929         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 930                 goto err_free_cpumask;
 931
 932         INIT_LIST_HEAD(&policy->policy_list);
 933         return policy;
 934
 935 err_free_cpumask:
 936         free_cpumask_var(policy->cpus);
 937 err_free_policy:
 938         kfree(policy);
 939
 940         return NULL;
 941 }
 942
 943 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 944 {
 945         free_cpumask_var(policy->related_cpus);
 946         free_cpumask_var(policy->cpus);
 947         kfree(policy);
 948 }
 949
 950 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
 951 {
 952         if (cpu == policy->cpu)
 953                 return;
 954
 955         policy->last_cpu = policy->cpu;
 956         policy->cpu = cpu;
 957
 958 #ifdef CONFIG_CPU_FREQ_TABLE
 959         cpufreq_frequency_table_update_policy_cpu(policy);
 960 #endif
 961         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 962                         CPUFREQ_UPDATE_POLICY_CPU, policy);
 963 }
 964
 965 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
 966                              bool frozen)
 967 {
 968         unsigned int j, cpu = dev->id;
 969         int ret = -ENOMEM;
 970         struct cpufreq_policy *policy;
 971         unsigned long flags;
 972 #ifdef CONFIG_HOTPLUG_CPU
 973         struct cpufreq_policy *tpolicy;
 974         struct cpufreq_governor *gov;
 975 #endif
 976
 977         if (cpu_is_offline(cpu))
 978                 return 0;
 979
 980         pr_debug("adding CPU %u\n", cpu);
 981
 982 #ifdef CONFIG_SMP
 983         /* check whether a different CPU already registered this
 984          * CPU because it is in the same boat. */
 985         policy = cpufreq_cpu_get(cpu);
 986         if (unlikely(policy)) {
 987                 cpufreq_cpu_put(policy);
 988                 return 0;
 989         }
 990 #endif
 991
 992         if (!down_read_trylock(&cpufreq_rwsem))
 993                 return 0;
 994
 995 #ifdef CONFIG_HOTPLUG_CPU
 996         /* Check if this cpu was hot-unplugged earlier and has siblings */
 997         read_lock_irqsave(&cpufreq_driver_lock, flags);
 998         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
 999                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1000                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1001                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev, frozen);
1002                         up_read(&cpufreq_rwsem);
1003                         return ret;
1004                 }
1005         }
1006         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1007 #endif
1008
1009         if (frozen)
1010                 /* Restore the saved policy when doing light-weight init */
1011                 policy = cpufreq_policy_restore(cpu);
1012         else
1013                 policy = cpufreq_policy_alloc();
1014
1015         if (!policy)
1016                 goto nomem_out;
1017
1018
1019         /*
1020          * In the resume path, since we restore a saved policy, the assignment
1021          * to policy->cpu is like an update of the existing policy, rather than
1022          * the creation of a brand new one. So we need to perform this update
1023          * by invoking update_policy_cpu().
1024          */
1025         if (frozen && cpu != policy->cpu)
1026                 update_policy_cpu(policy, cpu);
1027         else
1028                 policy->cpu = cpu;
1029
1030         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1031         cpumask_copy(policy->cpus, cpumask_of(cpu));
1032
1033         init_completion(&policy->kobj_unregister);
1034         INIT_WORK(&policy->update, handle_update);
1035
1036         /* call driver. From then on the cpufreq must be able
1037          * to accept all calls to ->verify and ->setpolicy for this CPU
1038          */
1039         ret = cpufreq_driver->init(policy);
1040         if (ret) {
1041                 pr_debug("initialization failed\n");
1042                 goto err_set_policy_cpu;
1043         }
1044
1045         /* related cpus should atleast have policy->cpus */
1046         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1047
1048         /*
1049          * affected cpus must always be the one, which are online. We aren't
1050          * managing offline cpus here.
1051          */
1052         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1053
1054         policy->user_policy.min = policy->min;
1055         policy->user_policy.max = policy->max;
1056
1057         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1058                                      CPUFREQ_START, policy);
1059
1060 #ifdef CONFIG_HOTPLUG_CPU
1061         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1062         if (gov) {
1063                 policy->governor = gov;
1064                 pr_debug("Restoring governor %s for cpu %d\n",
1065                        policy->governor->name, cpu);
1066         }
1067 #endif
1068
1069         write_lock_irqsave(&cpufreq_driver_lock, flags);
1070         for_each_cpu(j, policy->cpus)
1071                 per_cpu(cpufreq_cpu_data, j) = policy;
1072         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1073
1074         if (!frozen) {
1075                 ret = cpufreq_add_dev_interface(policy, dev);
1076                 if (ret)
1077                         goto err_out_unregister;
1078         }
1079
1080         write_lock_irqsave(&cpufreq_driver_lock, flags);
1081         list_add(&policy->policy_list, &cpufreq_policy_list);
1082         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1083
1084         cpufreq_init_policy(policy);
1085
1086         kobject_uevent(&policy->kobj, KOBJ_ADD);
1087         up_read(&cpufreq_rwsem);
1088
1089         pr_debug("initialization complete\n");
1090
1091         return 0;
1092
1093 err_out_unregister:
1094         write_lock_irqsave(&cpufreq_driver_lock, flags);
1095         for_each_cpu(j, policy->cpus)
1096                 per_cpu(cpufreq_cpu_data, j) = NULL;
1097         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1098
1099 err_set_policy_cpu:
1100         cpufreq_policy_free(policy);
1101 nomem_out:
1102         up_read(&cpufreq_rwsem);
1103
1104         return ret;
1105 }
1106
1107 /**
1108  * cpufreq_add_dev - add a CPU device
1109  *
1110  * Adds the cpufreq interface for a CPU device.
1111  *
1112  * The Oracle says: try running cpufreq registration/unregistration concurrently
1113  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1114  * mess up, but more thorough testing is needed. - Mathieu
1115  */
1116 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1117 {
1118         return __cpufreq_add_dev(dev, sif, false);
1119 }
1120
1121 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1122                                            unsigned int old_cpu, bool frozen)
1123 {
1124         struct device *cpu_dev;
1125         int ret;
1126
1127         /* first sibling now owns the new sysfs dir */
1128         cpu_dev = get_cpu_device(cpumask_first(policy->cpus));
1129
1130         /* Don't touch sysfs files during light-weight tear-down */
1131         if (frozen)
1132                 return cpu_dev->id;
1133
1134         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1135         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1136         if (ret) {
1137                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1138
1139                 WARN_ON(lock_policy_rwsem_write(old_cpu));
1140                 cpumask_set_cpu(old_cpu, policy->cpus);
1141                 unlock_policy_rwsem_write(old_cpu);
1142
1143                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1144                                         "cpufreq");
1145
1146                 return -EINVAL;
1147         }
1148
1149         return cpu_dev->id;
1150 }
1151
1152 static int __cpufreq_remove_dev_prepare(struct device *dev,
1153                                         struct subsys_interface *sif,
1154                                         bool frozen)
1155 {
1156         unsigned int cpu = dev->id, cpus;
1157         int new_cpu, ret;
1158         unsigned long flags;
1159         struct cpufreq_policy *policy;
1160
1161         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1162
1163         write_lock_irqsave(&cpufreq_driver_lock, flags);
1164
1165         policy = per_cpu(cpufreq_cpu_data, cpu);
1166
1167         /* Save the policy somewhere when doing a light-weight tear-down */
1168         if (frozen)
1169                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1170
1171         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1172
1173         if (!policy) {
1174                 pr_debug("%s: No cpu_data found\n", __func__);
1175                 return -EINVAL;
1176         }
1177
1178         if (cpufreq_driver->target) {
1179                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1180                 if (ret) {
1181                         pr_err("%s: Failed to stop governor\n", __func__);
1182                         return ret;
1183                 }
1184         }
1185
1186 #ifdef CONFIG_HOTPLUG_CPU
1187         if (!cpufreq_driver->setpolicy)
1188                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1189                         policy->governor->name, CPUFREQ_NAME_LEN);
1190 #endif
1191
1192         WARN_ON(lock_policy_rwsem_write(cpu));
1193         cpus = cpumask_weight(policy->cpus);
1194
1195         if (cpus > 1)
1196                 cpumask_clear_cpu(cpu, policy->cpus);
1197         unlock_policy_rwsem_write(cpu);
1198
1199         if (cpu != policy->cpu) {
1200                 if (!frozen)
1201                         sysfs_remove_link(&dev->kobj, "cpufreq");
1202         } else if (cpus > 1) {
1203
1204                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu, frozen);
1205                 if (new_cpu >= 0) {
1206                         WARN_ON(lock_policy_rwsem_write(cpu));
1207                         update_policy_cpu(policy, new_cpu);
1208                         unlock_policy_rwsem_write(cpu);
1209
1210                         if (!frozen) {
1211                                 pr_debug("%s: policy Kobject moved to cpu: %d "
1212                                          "from: %d\n",__func__, new_cpu, cpu);
1213                         }
1214                 }
1215         }
1216
1217         return 0;
1218 }
1219
1220 static int __cpufreq_remove_dev_finish(struct device *dev,
1221                                        struct subsys_interface *sif,
1222                                        bool frozen)
1223 {
1224         unsigned int cpu = dev->id, cpus;
1225         int ret;
1226         unsigned long flags;
1227         struct cpufreq_policy *policy;
1228         struct kobject *kobj;
1229         struct completion *cmp;
1230
1231         read_lock_irqsave(&cpufreq_driver_lock, flags);
1232         policy = per_cpu(cpufreq_cpu_data, cpu);
1233         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1234
1235         if (!policy) {
1236                 pr_debug("%s: No cpu_data found\n", __func__);
1237                 return -EINVAL;
1238         }
1239
1240         lock_policy_rwsem_read(cpu);
1241         cpus = cpumask_weight(policy->cpus);
1242         unlock_policy_rwsem_read(cpu);
1243
1244         /* If cpu is last user of policy, free policy */
1245         if (cpus == 1) {
1246                 if (cpufreq_driver->target) {
1247                         ret = __cpufreq_governor(policy,
1248                                         CPUFREQ_GOV_POLICY_EXIT);
1249                         if (ret) {
1250                                 pr_err("%s: Failed to exit governor\n",
1251                                                 __func__);
1252                                 return ret;
1253                         }
1254                 }
1255
1256                 if (!frozen) {
1257                         lock_policy_rwsem_read(cpu);
1258                         kobj = &policy->kobj;
1259                         cmp = &policy->kobj_unregister;
1260                         unlock_policy_rwsem_read(cpu);
1261                         kobject_put(kobj);
1262
1263                         /*
1264                          * We need to make sure that the underlying kobj is
1265                          * actually not referenced anymore by anybody before we
1266                          * proceed with unloading.
1267                          */
1268                         pr_debug("waiting for dropping of refcount\n");
1269                         wait_for_completion(cmp);
1270                         pr_debug("wait complete\n");
1271                 }
1272
1273                 /*
1274                  * Perform the ->exit() even during light-weight tear-down,
1275                  * since this is a core component, and is essential for the
1276                  * subsequent light-weight ->init() to succeed.
1277                  */
1278                 if (cpufreq_driver->exit)
1279                         cpufreq_driver->exit(policy);
1280
1281                 /* Remove policy from list of active policies */
1282                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1283                 list_del(&policy->policy_list);
1284                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1285
1286                 if (!frozen)
1287                         cpufreq_policy_free(policy);
1288         } else {
1289                 if (cpufreq_driver->target) {
1290                         if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
1291                                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
1292                                 pr_err("%s: Failed to start governor\n",
1293                                                 __func__);
1294                                 return ret;
1295                         }
1296                 }
1297         }
1298
1299         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1300         return 0;
1301 }
1302
1303 /**
1304  * __cpufreq_remove_dev - remove a CPU device
1305  *
1306  * Removes the cpufreq interface for a CPU device.
1307  * Caller should already have policy_rwsem in write mode for this CPU.
1308  * This routine frees the rwsem before returning.
1309  */
1310 static inline int __cpufreq_remove_dev(struct device *dev,
1311                                        struct subsys_interface *sif,
1312                                        bool frozen)
1313 {
1314         int ret;
1315
1316         ret = __cpufreq_remove_dev_prepare(dev, sif, frozen);
1317
1318         if (!ret)
1319                 ret = __cpufreq_remove_dev_finish(dev, sif, frozen);
1320
1321         return ret;
1322 }
1323
1324 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1325 {
1326         unsigned int cpu = dev->id;
1327         int retval;
1328
1329         if (cpu_is_offline(cpu))
1330                 return 0;
1331
1332         retval = __cpufreq_remove_dev(dev, sif, false);
1333         return retval;
1334 }
1335
1336 static void handle_update(struct work_struct *work)
1337 {
1338         struct cpufreq_policy *policy =
1339                 container_of(work, struct cpufreq_policy, update);
1340         unsigned int cpu = policy->cpu;
1341         pr_debug("handle_update for cpu %u called\n", cpu);
1342         cpufreq_update_policy(cpu);
1343 }
1344
1345 /**
1346  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1347  *      in deep trouble.
1348  *      @cpu: cpu number
1349  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1350  *      @new_freq: CPU frequency the CPU actually runs at
1351  *
1352  *      We adjust to current frequency first, and need to clean up later.
1353  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1354  */
1355 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1356                                 unsigned int new_freq)
1357 {
1358         struct cpufreq_policy *policy;
1359         struct cpufreq_freqs freqs;
1360         unsigned long flags;
1361
1362         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1363                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1364
1365         freqs.old = old_freq;
1366         freqs.new = new_freq;
1367
1368         read_lock_irqsave(&cpufreq_driver_lock, flags);
1369         policy = per_cpu(cpufreq_cpu_data, cpu);
1370         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1371
1372         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1373         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1374 }
1375
1376 /**
1377  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1378  * @cpu: CPU number
1379  *
1380  * This is the last known freq, without actually getting it from the driver.
1381  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1382  */
1383 unsigned int cpufreq_quick_get(unsigned int cpu)
1384 {
1385         struct cpufreq_policy *policy;
1386         unsigned int ret_freq = 0;
1387
1388         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1389                 return cpufreq_driver->get(cpu);
1390
1391         policy = cpufreq_cpu_get(cpu);
1392         if (policy) {
1393                 ret_freq = policy->cur;
1394                 cpufreq_cpu_put(policy);
1395         }
1396
1397         return ret_freq;
1398 }
1399 EXPORT_SYMBOL(cpufreq_quick_get);
1400
1401 /**
1402  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1403  * @cpu: CPU number
1404  *
1405  * Just return the max possible frequency for a given CPU.
1406  */
1407 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1408 {
1409         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1410         unsigned int ret_freq = 0;
1411
1412         if (policy) {
1413                 ret_freq = policy->max;
1414                 cpufreq_cpu_put(policy);
1415         }
1416
1417         return ret_freq;
1418 }
1419 EXPORT_SYMBOL(cpufreq_quick_get_max);
1420
1421 static unsigned int __cpufreq_get(unsigned int cpu)
1422 {
1423         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1424         unsigned int ret_freq = 0;
1425
1426         if (!cpufreq_driver->get)
1427                 return ret_freq;
1428
1429         ret_freq = cpufreq_driver->get(cpu);
1430
1431         if (ret_freq && policy->cur &&
1432                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1433                 /* verify no discrepancy between actual and
1434                                         saved value exists */
1435                 if (unlikely(ret_freq != policy->cur)) {
1436                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1437                         schedule_work(&policy->update);
1438                 }
1439         }
1440
1441         return ret_freq;
1442 }
1443
1444 /**
1445  * cpufreq_get - get the current CPU frequency (in kHz)
1446  * @cpu: CPU number
1447  *
1448  * Get the CPU current (static) CPU frequency
1449  */
1450 unsigned int cpufreq_get(unsigned int cpu)
1451 {
1452         unsigned int ret_freq = 0;
1453
1454         if (!down_read_trylock(&cpufreq_rwsem))
1455                 return 0;
1456
1457         if (unlikely(lock_policy_rwsem_read(cpu)))
1458                 goto out_policy;
1459
1460         ret_freq = __cpufreq_get(cpu);
1461
1462         unlock_policy_rwsem_read(cpu);
1463
1464 out_policy:
1465         up_read(&cpufreq_rwsem);
1466
1467         return ret_freq;
1468 }
1469 EXPORT_SYMBOL(cpufreq_get);
1470
1471 static struct subsys_interface cpufreq_interface = {
1472         .name           = "cpufreq",
1473         .subsys         = &cpu_subsys,
1474         .add_dev        = cpufreq_add_dev,
1475         .remove_dev     = cpufreq_remove_dev,
1476 };
1477
1478 /**
1479  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1480  *
1481  * This function is only executed for the boot processor.  The other CPUs
1482  * have been put offline by means of CPU hotplug.
1483  */
1484 static int cpufreq_bp_suspend(void)
1485 {
1486         int ret = 0;
1487
1488         int cpu = smp_processor_id();
1489         struct cpufreq_policy *policy;
1490
1491         pr_debug("suspending cpu %u\n", cpu);
1492
1493         /* If there's no policy for the boot CPU, we have nothing to do. */
1494         policy = cpufreq_cpu_get(cpu);
1495         if (!policy)
1496                 return 0;
1497
1498         if (cpufreq_driver->suspend) {
1499                 ret = cpufreq_driver->suspend(policy);
1500                 if (ret)
1501                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1502                                         "step on CPU %u\n", policy->cpu);
1503         }
1504
1505         cpufreq_cpu_put(policy);
1506         return ret;
1507 }
1508
1509 /**
1510  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1511  *
1512  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1513  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1514  *          restored. It will verify that the current freq is in sync with
1515  *          what we believe it to be. This is a bit later than when it
1516  *          should be, but nonethteless it's better than calling
1517  *          cpufreq_driver->get() here which might re-enable interrupts...
1518  *
1519  * This function is only executed for the boot CPU.  The other CPUs have not
1520  * been turned on yet.
1521  */
1522 static void cpufreq_bp_resume(void)
1523 {
1524         int ret = 0;
1525
1526         int cpu = smp_processor_id();
1527         struct cpufreq_policy *policy;
1528
1529         pr_debug("resuming cpu %u\n", cpu);
1530
1531         /* If there's no policy for the boot CPU, we have nothing to do. */
1532         policy = cpufreq_cpu_get(cpu);
1533         if (!policy)
1534                 return;
1535
1536         if (cpufreq_driver->resume) {
1537                 ret = cpufreq_driver->resume(policy);
1538                 if (ret) {
1539                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1540                                         "step on CPU %u\n", policy->cpu);
1541                         goto fail;
1542                 }
1543         }
1544
1545         schedule_work(&policy->update);
1546
1547 fail:
1548         cpufreq_cpu_put(policy);
1549 }
1550
1551 static struct syscore_ops cpufreq_syscore_ops = {
1552         .suspend        = cpufreq_bp_suspend,
1553         .resume         = cpufreq_bp_resume,
1554 };
1555
1556 /**
1557  *      cpufreq_get_current_driver - return current driver's name
1558  *
1559  *      Return the name string of the currently loaded cpufreq driver
1560  *      or NULL, if none.
1561  */
1562 const char *cpufreq_get_current_driver(void)
1563 {
1564         if (cpufreq_driver)
1565                 return cpufreq_driver->name;
1566
1567         return NULL;
1568 }
1569 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1570
1571 /*********************************************************************
1572  *                     NOTIFIER LISTS INTERFACE                      *
1573  *********************************************************************/
1574
1575 /**
1576  *      cpufreq_register_notifier - register a driver with cpufreq
1577  *      @nb: notifier function to register
1578  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1579  *
1580  *      Add a driver to one of two lists: either a list of drivers that
1581  *      are notified about clock rate changes (once before and once after
1582  *      the transition), or a list of drivers that are notified about
1583  *      changes in cpufreq policy.
1584  *
1585  *      This function may sleep, and has the same return conditions as
1586  *      blocking_notifier_chain_register.
1587  */
1588 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1589 {
1590         int ret;
1591
1592         if (cpufreq_disabled())
1593                 return -EINVAL;
1594
1595         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1596
1597         switch (list) {
1598         case CPUFREQ_TRANSITION_NOTIFIER:
1599                 ret = srcu_notifier_chain_register(
1600                                 &cpufreq_transition_notifier_list, nb);
1601                 break;
1602         case CPUFREQ_POLICY_NOTIFIER:
1603                 ret = blocking_notifier_chain_register(
1604                                 &cpufreq_policy_notifier_list, nb);
1605                 break;
1606         default:
1607                 ret = -EINVAL;
1608         }
1609
1610         return ret;
1611 }
1612 EXPORT_SYMBOL(cpufreq_register_notifier);
1613
1614 /**
1615  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1616  *      @nb: notifier block to be unregistered
1617  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1618  *
1619  *      Remove a driver from the CPU frequency notifier list.
1620  *
1621  *      This function may sleep, and has the same return conditions as
1622  *      blocking_notifier_chain_unregister.
1623  */
1624 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1625 {
1626         int ret;
1627
1628         if (cpufreq_disabled())
1629                 return -EINVAL;
1630
1631         switch (list) {
1632         case CPUFREQ_TRANSITION_NOTIFIER:
1633                 ret = srcu_notifier_chain_unregister(
1634                                 &cpufreq_transition_notifier_list, nb);
1635                 break;
1636         case CPUFREQ_POLICY_NOTIFIER:
1637                 ret = blocking_notifier_chain_unregister(
1638                                 &cpufreq_policy_notifier_list, nb);
1639                 break;
1640         default:
1641                 ret = -EINVAL;
1642         }
1643
1644         return ret;
1645 }
1646 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1647
1648
1649 /*********************************************************************
1650  *                              GOVERNORS                            *
1651  *********************************************************************/
1652
1653 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1654                             unsigned int target_freq,
1655                             unsigned int relation)
1656 {
1657         int retval = -EINVAL;
1658         unsigned int old_target_freq = target_freq;
1659
1660         if (cpufreq_disabled())
1661                 return -ENODEV;
1662
1663         /* Make sure that target_freq is within supported range */
1664         if (target_freq > policy->max)
1665                 target_freq = policy->max;
1666         if (target_freq < policy->min)
1667                 target_freq = policy->min;
1668
1669         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1670                         policy->cpu, target_freq, relation, old_target_freq);
1671
1672         if (target_freq == policy->cur)
1673                 return 0;
1674
1675         if (cpufreq_driver->target)
1676                 retval = cpufreq_driver->target(policy, target_freq, relation);
1677
1678         return retval;
1679 }
1680 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1681
1682 int cpufreq_driver_target(struct cpufreq_policy *policy,
1683                           unsigned int target_freq,
1684                           unsigned int relation)
1685 {
1686         int ret = -EINVAL;
1687
1688         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1689                 goto fail;
1690
1691         ret = __cpufreq_driver_target(policy, target_freq, relation);
1692
1693         unlock_policy_rwsem_write(policy->cpu);
1694
1695 fail:
1696         return ret;
1697 }
1698 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1699
1700 /*
1701  * when "event" is CPUFREQ_GOV_LIMITS
1702  */
1703
1704 static int __cpufreq_governor(struct cpufreq_policy *policy,
1705                                         unsigned int event)
1706 {
1707         int ret;
1708
1709         /* Only must be defined when default governor is known to have latency
1710            restrictions, like e.g. conservative or ondemand.
1711            That this is the case is already ensured in Kconfig
1712         */
1713 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1714         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1715 #else
1716         struct cpufreq_governor *gov = NULL;
1717 #endif
1718
1719         if (policy->governor->max_transition_latency &&
1720             policy->cpuinfo.transition_latency >
1721             policy->governor->max_transition_latency) {
1722                 if (!gov)
1723                         return -EINVAL;
1724                 else {
1725                         printk(KERN_WARNING "%s governor failed, too long"
1726                                " transition latency of HW, fallback"
1727                                " to %s governor\n",
1728                                policy->governor->name,
1729                                gov->name);
1730                         policy->governor = gov;
1731                 }
1732         }
1733
1734         if (event == CPUFREQ_GOV_POLICY_INIT)
1735                 if (!try_module_get(policy->governor->owner))
1736                         return -EINVAL;
1737
1738         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1739                                                 policy->cpu, event);
1740
1741         mutex_lock(&cpufreq_governor_lock);
1742         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1743             || (!policy->governor_enabled
1744             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1745                 mutex_unlock(&cpufreq_governor_lock);
1746                 return -EBUSY;
1747         }
1748
1749         if (event == CPUFREQ_GOV_STOP)
1750                 policy->governor_enabled = false;
1751         else if (event == CPUFREQ_GOV_START)
1752                 policy->governor_enabled = true;
1753
1754         mutex_unlock(&cpufreq_governor_lock);
1755
1756         ret = policy->governor->governor(policy, event);
1757
1758         if (!ret) {
1759                 if (event == CPUFREQ_GOV_POLICY_INIT)
1760                         policy->governor->initialized++;
1761                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1762                         policy->governor->initialized--;
1763         } else {
1764                 /* Restore original values */
1765                 mutex_lock(&cpufreq_governor_lock);
1766                 if (event == CPUFREQ_GOV_STOP)
1767                         policy->governor_enabled = true;
1768                 else if (event == CPUFREQ_GOV_START)
1769                         policy->governor_enabled = false;
1770                 mutex_unlock(&cpufreq_governor_lock);
1771         }
1772
1773         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1774                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1775                 module_put(policy->governor->owner);
1776
1777         return ret;
1778 }
1779
1780 int cpufreq_register_governor(struct cpufreq_governor *governor)
1781 {
1782         int err;
1783
1784         if (!governor)
1785                 return -EINVAL;
1786
1787         if (cpufreq_disabled())
1788                 return -ENODEV;
1789
1790         mutex_lock(&cpufreq_governor_mutex);
1791
1792         governor->initialized = 0;
1793         err = -EBUSY;
1794         if (__find_governor(governor->name) == NULL) {
1795                 err = 0;
1796                 list_add(&governor->governor_list, &cpufreq_governor_list);
1797         }
1798
1799         mutex_unlock(&cpufreq_governor_mutex);
1800         return err;
1801 }
1802 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1803
1804 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1805 {
1806 #ifdef CONFIG_HOTPLUG_CPU
1807         int cpu;
1808 #endif
1809
1810         if (!governor)
1811                 return;
1812
1813         if (cpufreq_disabled())
1814                 return;
1815
1816 #ifdef CONFIG_HOTPLUG_CPU
1817         for_each_present_cpu(cpu) {
1818                 if (cpu_online(cpu))
1819                         continue;
1820                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1821                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1822         }
1823 #endif
1824
1825         mutex_lock(&cpufreq_governor_mutex);
1826         list_del(&governor->governor_list);
1827         mutex_unlock(&cpufreq_governor_mutex);
1828         return;
1829 }
1830 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1831
1832
1833 /*********************************************************************
1834  *                          POLICY INTERFACE                         *
1835  *********************************************************************/
1836
1837 /**
1838  * cpufreq_get_policy - get the current cpufreq_policy
1839  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1840  *      is written
1841  *
1842  * Reads the current cpufreq policy.
1843  */
1844 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1845 {
1846         struct cpufreq_policy *cpu_policy;
1847         if (!policy)
1848                 return -EINVAL;
1849
1850         cpu_policy = cpufreq_cpu_get(cpu);
1851         if (!cpu_policy)
1852                 return -EINVAL;
1853
1854         memcpy(policy, cpu_policy, sizeof(*policy));
1855
1856         cpufreq_cpu_put(cpu_policy);
1857         return 0;
1858 }
1859 EXPORT_SYMBOL(cpufreq_get_policy);
1860
1861 /*
1862  * data   : current policy.
1863  * policy : policy to be set.
1864  */
1865 static int __cpufreq_set_policy(struct cpufreq_policy *policy,
1866                                 struct cpufreq_policy *new_policy)
1867 {
1868         int ret = 0, failed = 1;
1869
1870         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
1871                 new_policy->min, new_policy->max);
1872
1873         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
1874
1875         if (new_policy->min > policy->max || new_policy->max < policy->min) {
1876                 ret = -EINVAL;
1877                 goto error_out;
1878         }
1879
1880         /* verify the cpu speed can be set within this limit */
1881         ret = cpufreq_driver->verify(new_policy);
1882         if (ret)
1883                 goto error_out;
1884
1885         /* adjust if necessary - all reasons */
1886         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1887                         CPUFREQ_ADJUST, new_policy);
1888
1889         /* adjust if necessary - hardware incompatibility*/
1890         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1891                         CPUFREQ_INCOMPATIBLE, new_policy);
1892
1893         /*
1894          * verify the cpu speed can be set within this limit, which might be
1895          * different to the first one
1896          */
1897         ret = cpufreq_driver->verify(new_policy);
1898         if (ret)
1899                 goto error_out;
1900
1901         /* notification of the new policy */
1902         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1903                         CPUFREQ_NOTIFY, new_policy);
1904
1905         policy->min = new_policy->min;
1906         policy->max = new_policy->max;
1907
1908         pr_debug("new min and max freqs are %u - %u kHz\n",
1909                                         policy->min, policy->max);
1910
1911         if (cpufreq_driver->setpolicy) {
1912                 policy->policy = new_policy->policy;
1913                 pr_debug("setting range\n");
1914                 ret = cpufreq_driver->setpolicy(new_policy);
1915         } else {
1916                 if (new_policy->governor != policy->governor) {
1917                         /* save old, working values */
1918                         struct cpufreq_governor *old_gov = policy->governor;
1919
1920                         pr_debug("governor switch\n");
1921
1922                         /* end old governor */
1923                         if (policy->governor) {
1924                                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1925                                 unlock_policy_rwsem_write(new_policy->cpu);
1926                                 __cpufreq_governor(policy,
1927                                                 CPUFREQ_GOV_POLICY_EXIT);
1928                                 lock_policy_rwsem_write(new_policy->cpu);
1929                         }
1930
1931                         /* start new governor */
1932                         policy->governor = new_policy->governor;
1933                         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
1934                                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
1935                                         failed = 0;
1936                                 } else {
1937                                         unlock_policy_rwsem_write(new_policy->cpu);
1938                                         __cpufreq_governor(policy,
1939                                                         CPUFREQ_GOV_POLICY_EXIT);
1940                                         lock_policy_rwsem_write(new_policy->cpu);
1941                                 }
1942                         }
1943
1944                         if (failed) {
1945                                 /* new governor failed, so re-start old one */
1946                                 pr_debug("starting governor %s failed\n",
1947                                                         policy->governor->name);
1948                                 if (old_gov) {
1949                                         policy->governor = old_gov;
1950                                         __cpufreq_governor(policy,
1951                                                         CPUFREQ_GOV_POLICY_INIT);
1952                                         __cpufreq_governor(policy,
1953                                                            CPUFREQ_GOV_START);
1954                                 }
1955                                 ret = -EINVAL;
1956                                 goto error_out;
1957                         }
1958                         /* might be a policy change, too, so fall through */
1959                 }
1960                 pr_debug("governor: change or update limits\n");
1961                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1962         }
1963
1964 error_out:
1965         return ret;
1966 }
1967
1968 /**
1969  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1970  *      @cpu: CPU which shall be re-evaluated
1971  *
1972  *      Useful for policy notifiers which have different necessities
1973  *      at different times.
1974  */
1975 int cpufreq_update_policy(unsigned int cpu)
1976 {
1977         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1978         struct cpufreq_policy new_policy;
1979         int ret;
1980
1981         if (!policy) {
1982                 ret = -ENODEV;
1983                 goto no_policy;
1984         }
1985
1986         if (unlikely(lock_policy_rwsem_write(cpu))) {
1987                 ret = -EINVAL;
1988                 goto fail;
1989         }
1990
1991         pr_debug("updating policy for CPU %u\n", cpu);
1992         memcpy(&new_policy, policy, sizeof(*policy));
1993         new_policy.min = policy->user_policy.min;
1994         new_policy.max = policy->user_policy.max;
1995         new_policy.policy = policy->user_policy.policy;
1996         new_policy.governor = policy->user_policy.governor;
1997
1998         /*
1999          * BIOS might change freq behind our back
2000          * -> ask driver for current freq and notify governors about a change
2001          */
2002         if (cpufreq_driver->get) {
2003                 new_policy.cur = cpufreq_driver->get(cpu);
2004                 if (!policy->cur) {
2005                         pr_debug("Driver did not initialize current freq");
2006                         policy->cur = new_policy.cur;
2007                 } else {
2008                         if (policy->cur != new_policy.cur && cpufreq_driver->target)
2009                                 cpufreq_out_of_sync(cpu, policy->cur,
2010                                                                 new_policy.cur);
2011                 }
2012         }
2013
2014         ret = __cpufreq_set_policy(policy, &new_policy);
2015
2016         unlock_policy_rwsem_write(cpu);
2017
2018 fail:
2019         cpufreq_cpu_put(policy);
2020 no_policy:
2021         return ret;
2022 }
2023 EXPORT_SYMBOL(cpufreq_update_policy);
2024
2025 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2026                                         unsigned long action, void *hcpu)
2027 {
2028         unsigned int cpu = (unsigned long)hcpu;
2029         struct device *dev;
2030         bool frozen = false;
2031
2032         dev = get_cpu_device(cpu);
2033         if (dev) {
2034
2035                 if (action & CPU_TASKS_FROZEN)
2036                         frozen = true;
2037
2038                 switch (action & ~CPU_TASKS_FROZEN) {
2039                 case CPU_ONLINE:
2040                         __cpufreq_add_dev(dev, NULL, frozen);
2041                         cpufreq_update_policy(cpu);
2042                         break;
2043
2044                 case CPU_DOWN_PREPARE:
2045                         __cpufreq_remove_dev_prepare(dev, NULL, frozen);
2046                         break;
2047
2048                 case CPU_POST_DEAD:
2049                         __cpufreq_remove_dev_finish(dev, NULL, frozen);
2050                         break;
2051
2052                 case CPU_DOWN_FAILED:
2053                         __cpufreq_add_dev(dev, NULL, frozen);
2054                         break;
2055                 }
2056         }
2057         return NOTIFY_OK;
2058 }
2059
2060 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2061         .notifier_call = cpufreq_cpu_callback,
2062 };
2063
2064 /*********************************************************************
2065  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2066  *********************************************************************/
2067
2068 /**
2069  * cpufreq_register_driver - register a CPU Frequency driver
2070  * @driver_data: A struct cpufreq_driver containing the values#
2071  * submitted by the CPU Frequency driver.
2072  *
2073  * Registers a CPU Frequency driver to this core code. This code
2074  * returns zero on success, -EBUSY when another driver got here first
2075  * (and isn't unregistered in the meantime).
2076  *
2077  */
2078 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2079 {
2080         unsigned long flags;
2081         int ret;
2082
2083         if (cpufreq_disabled())
2084                 return -ENODEV;
2085
2086         if (!driver_data || !driver_data->verify || !driver_data->init ||
2087             ((!driver_data->setpolicy) && (!driver_data->target)))
2088                 return -EINVAL;
2089
2090         pr_debug("trying to register driver %s\n", driver_data->name);
2091
2092         if (driver_data->setpolicy)
2093                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2094
2095         write_lock_irqsave(&cpufreq_driver_lock, flags);
2096         if (cpufreq_driver) {
2097                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2098                 return -EBUSY;
2099         }
2100         cpufreq_driver = driver_data;
2101         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2102
2103         ret = subsys_interface_register(&cpufreq_interface);
2104         if (ret)
2105                 goto err_null_driver;
2106
2107         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2108                 int i;
2109                 ret = -ENODEV;
2110
2111                 /* check for at least one working CPU */
2112                 for (i = 0; i < nr_cpu_ids; i++)
2113                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2114                                 ret = 0;
2115                                 break;
2116                         }
2117
2118                 /* if all ->init() calls failed, unregister */
2119                 if (ret) {
2120                         pr_debug("no CPU initialized for driver %s\n",
2121                                                         driver_data->name);
2122                         goto err_if_unreg;
2123                 }
2124         }
2125
2126         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2127         pr_debug("driver %s up and running\n", driver_data->name);
2128
2129         return 0;
2130 err_if_unreg:
2131         subsys_interface_unregister(&cpufreq_interface);
2132 err_null_driver:
2133         write_lock_irqsave(&cpufreq_driver_lock, flags);
2134         cpufreq_driver = NULL;
2135         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2136         return ret;
2137 }
2138 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2139
2140 /**
2141  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2142  *
2143  * Unregister the current CPUFreq driver. Only call this if you have
2144  * the right to do so, i.e. if you have succeeded in initialising before!
2145  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2146  * currently not initialised.
2147  */
2148 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2149 {
2150         unsigned long flags;
2151
2152         if (!cpufreq_driver || (driver != cpufreq_driver))
2153                 return -EINVAL;
2154
2155         pr_debug("unregistering driver %s\n", driver->name);
2156
2157         subsys_interface_unregister(&cpufreq_interface);
2158         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2159
2160         down_write(&cpufreq_rwsem);
2161         write_lock_irqsave(&cpufreq_driver_lock, flags);
2162
2163         cpufreq_driver = NULL;
2164
2165         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2166         up_write(&cpufreq_rwsem);
2167
2168         return 0;
2169 }
2170 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2171
2172 static int __init cpufreq_core_init(void)
2173 {
2174         int cpu;
2175
2176         if (cpufreq_disabled())
2177                 return -ENODEV;
2178
2179         for_each_possible_cpu(cpu)
2180                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2181
2182         cpufreq_global_kobject = kobject_create();
2183         BUG_ON(!cpufreq_global_kobject);
2184         register_syscore_ops(&cpufreq_syscore_ops);
2185
2186         return 0;
2187 }
2188 core_initcall(cpufreq_core_init);