drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 /* This one keeps track of the previously set governor of a removed CPU */
  46 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  47
  48 /* Flag to suspend/resume CPUFreq governors */
  49 static bool cpufreq_suspended;
  50
  51 static inline bool has_target(void)
  52 {
  53         return cpufreq_driver->target_index || cpufreq_driver->target;
  54 }
  55
  56 /*
  57  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  58  * sections
  59  */
  60 static DECLARE_RWSEM(cpufreq_rwsem);
  61
  62 /* internal prototypes */
  63 static int __cpufreq_governor(struct cpufreq_policy *policy,
  64                 unsigned int event);
  65 static unsigned int __cpufreq_get(unsigned int cpu);
  66 static void handle_update(struct work_struct *work);
  67
  68 /**
  69  * Two notifier lists: the "policy" list is involved in the
  70  * validation process for a new CPU frequency policy; the
  71  * "transition" list for kernel code that needs to handle
  72  * changes to devices when the CPU clock speed changes.
  73  * The mutex locks both lists.
  74  */
  75 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  76 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  77
  78 static bool init_cpufreq_transition_notifier_list_called;
  79 static int __init init_cpufreq_transition_notifier_list(void)
  80 {
  81         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  82         init_cpufreq_transition_notifier_list_called = true;
  83         return 0;
  84 }
  85 pure_initcall(init_cpufreq_transition_notifier_list);
  86
  87 static int off __read_mostly;
  88 static int cpufreq_disabled(void)
  89 {
  90         return off;
  91 }
  92 void disable_cpufreq(void)
  93 {
  94         off = 1;
  95 }
  96 static LIST_HEAD(cpufreq_governor_list);
  97 static DEFINE_MUTEX(cpufreq_governor_mutex);
  98
  99 bool have_governor_per_policy(void)
 100 {
 101         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 102 }
 103 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 104
 105 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 106 {
 107         if (have_governor_per_policy())
 108                 return &policy->kobj;
 109         else
 110                 return cpufreq_global_kobject;
 111 }
 112 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 113
 114 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 115 {
 116         u64 idle_time;
 117         u64 cur_wall_time;
 118         u64 busy_time;
 119
 120         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 121
 122         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 127         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 128
 129         idle_time = cur_wall_time - busy_time;
 130         if (wall)
 131                 *wall = cputime_to_usecs(cur_wall_time);
 132
 133         return cputime_to_usecs(idle_time);
 134 }
 135
 136 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 137 {
 138         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 139
 140         if (idle_time == -1ULL)
 141                 return get_cpu_idle_time_jiffy(cpu, wall);
 142         else if (!io_busy)
 143                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 144
 145         return idle_time;
 146 }
 147 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 148
 149 /*
 150  * This is a generic cpufreq init() routine which can be used by cpufreq
 151  * drivers of SMP systems. It will do following:
 152  * - validate & show freq table passed
 153  * - set policies transition latency
 154  * - policy->cpus with all possible CPUs
 155  */
 156 int cpufreq_generic_init(struct cpufreq_policy *policy,
 157                 struct cpufreq_frequency_table *table,
 158                 unsigned int transition_latency)
 159 {
 160         int ret;
 161
 162         ret = cpufreq_table_validate_and_show(policy, table);
 163         if (ret) {
 164                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 165                 return ret;
 166         }
 167
 168         policy->cpuinfo.transition_latency = transition_latency;
 169
 170         /*
 171          * The driver only supports the SMP configuartion where all processors
 172          * share the clock and voltage and clock.
 173          */
 174         cpumask_setall(policy->cpus);
 175
 176         return 0;
 177 }
 178 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 179
 180 unsigned int cpufreq_generic_get(unsigned int cpu)
 181 {
 182         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 183
 184         if (!policy || IS_ERR(policy->clk)) {
 185                 pr_err("%s: No %s associated to cpu: %d\n",
 186                        __func__, policy ? "clk" : "policy", cpu);
 187                 return 0;
 188         }
 189
 190         return clk_get_rate(policy->clk) / 1000;
 191 }
 192 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 193
 194 /* Only for cpufreq core internal use */
 195 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 196 {
 197         return per_cpu(cpufreq_cpu_data, cpu);
 198 }
 199
 200 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 201 {
 202         struct cpufreq_policy *policy = NULL;
 203         unsigned long flags;
 204
 205         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 206                 return NULL;
 207
 208         if (!down_read_trylock(&cpufreq_rwsem))
 209                 return NULL;
 210
 211         /* get the cpufreq driver */
 212         read_lock_irqsave(&cpufreq_driver_lock, flags);
 213
 214         if (cpufreq_driver) {
 215                 /* get the CPU */
 216                 policy = per_cpu(cpufreq_cpu_data, cpu);
 217                 if (policy)
 218                         kobject_get(&policy->kobj);
 219         }
 220
 221         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 222
 223         if (!policy)
 224                 up_read(&cpufreq_rwsem);
 225
 226         return policy;
 227 }
 228 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 229
 230 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 231 {
 232         if (cpufreq_disabled())
 233                 return;
 234
 235         kobject_put(&policy->kobj);
 236         up_read(&cpufreq_rwsem);
 237 }
 238 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 239
 240 bool cpufreq_next_valid(struct cpufreq_frequency_table **pos)
 241 {
 242         while ((*pos)->frequency != CPUFREQ_TABLE_END)
 243                 if ((*pos)->frequency != CPUFREQ_ENTRY_INVALID)
 244                         return true;
 245                 else
 246                         (*pos)++;
 247         return false;
 248 }
 249 EXPORT_SYMBOL_GPL(cpufreq_next_valid);
 250
 251 /*********************************************************************
 252  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 253  *********************************************************************/
 254
 255 /**
 256  * adjust_jiffies - adjust the system "loops_per_jiffy"
 257  *
 258  * This function alters the system "loops_per_jiffy" for the clock
 259  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 260  * systems as each CPU might be scaled differently. So, use the arch
 261  * per-CPU loops_per_jiffy value wherever possible.
 262  */
 263 #ifndef CONFIG_SMP
 264 static unsigned long l_p_j_ref;
 265 static unsigned int l_p_j_ref_freq;
 266
 267 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 268 {
 269         if (ci->flags & CPUFREQ_CONST_LOOPS)
 270                 return;
 271
 272         if (!l_p_j_ref_freq) {
 273                 l_p_j_ref = loops_per_jiffy;
 274                 l_p_j_ref_freq = ci->old;
 275                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 276                          l_p_j_ref, l_p_j_ref_freq);
 277         }
 278         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 279                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 280                                                                 ci->new);
 281                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 282                          loops_per_jiffy, ci->new);
 283         }
 284 }
 285 #else
 286 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 287 {
 288         return;
 289 }
 290 #endif
 291
 292 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 293                 struct cpufreq_freqs *freqs, unsigned int state)
 294 {
 295         BUG_ON(irqs_disabled());
 296
 297         if (cpufreq_disabled())
 298                 return;
 299
 300         freqs->flags = cpufreq_driver->flags;
 301         pr_debug("notification %u of frequency transition to %u kHz\n",
 302                  state, freqs->new);
 303
 304         switch (state) {
 305
 306         case CPUFREQ_PRECHANGE:
 307                 /* detect if the driver reported a value as "old frequency"
 308                  * which is not equal to what the cpufreq core thinks is
 309                  * "old frequency".
 310                  */
 311                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 312                         if ((policy) && (policy->cpu == freqs->cpu) &&
 313                             (policy->cur) && (policy->cur != freqs->old)) {
 314                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 315                                          freqs->old, policy->cur);
 316                                 freqs->old = policy->cur;
 317                         }
 318                 }
 319                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 320                                 CPUFREQ_PRECHANGE, freqs);
 321                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 322                 break;
 323
 324         case CPUFREQ_POSTCHANGE:
 325                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 326                 pr_debug("FREQ: %lu - CPU: %lu\n",
 327                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 328                 trace_cpu_frequency(freqs->new, freqs->cpu);
 329                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 330                                 CPUFREQ_POSTCHANGE, freqs);
 331                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 332                         policy->cur = freqs->new;
 333                 break;
 334         }
 335 }
 336
 337 /**
 338  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 339  * on frequency transition.
 340  *
 341  * This function calls the transition notifiers and the "adjust_jiffies"
 342  * function. It is called twice on all CPU frequency changes that have
 343  * external effects.
 344  */
 345 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 346                 struct cpufreq_freqs *freqs, unsigned int state)
 347 {
 348         for_each_cpu(freqs->cpu, policy->cpus)
 349                 __cpufreq_notify_transition(policy, freqs, state);
 350 }
 351
 352 /* Do post notifications when there are chances that transition has failed */
 353 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 354                 struct cpufreq_freqs *freqs, int transition_failed)
 355 {
 356         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 357         if (!transition_failed)
 358                 return;
 359
 360         swap(freqs->old, freqs->new);
 361         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 362         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 363 }
 364
 365 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 366                 struct cpufreq_freqs *freqs)
 367 {
 368 wait:
 369         wait_event(policy->transition_wait, !policy->transition_ongoing);
 370
 371         spin_lock(&policy->transition_lock);
 372
 373         if (unlikely(policy->transition_ongoing)) {
 374                 spin_unlock(&policy->transition_lock);
 375                 goto wait;
 376         }
 377
 378         policy->transition_ongoing = true;
 379
 380         spin_unlock(&policy->transition_lock);
 381
 382         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 383 }
 384 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 385
 386 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 387                 struct cpufreq_freqs *freqs, int transition_failed)
 388 {
 389         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 390                 return;
 391
 392         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 393
 394         policy->transition_ongoing = false;
 395
 396         wake_up(&policy->transition_wait);
 397 }
 398 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 399
 400
 401 /*********************************************************************
 402  *                          SYSFS INTERFACE                          *
 403  *********************************************************************/
 404 static ssize_t show_boost(struct kobject *kobj,
 405                                  struct attribute *attr, char *buf)
 406 {
 407         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 408 }
 409
 410 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 411                                   const char *buf, size_t count)
 412 {
 413         int ret, enable;
 414
 415         ret = sscanf(buf, "%d", &enable);
 416         if (ret != 1 || enable < 0 || enable > 1)
 417                 return -EINVAL;
 418
 419         if (cpufreq_boost_trigger_state(enable)) {
 420                 pr_err("%s: Cannot %s BOOST!\n",
 421                        __func__, enable ? "enable" : "disable");
 422                 return -EINVAL;
 423         }
 424
 425         pr_debug("%s: cpufreq BOOST %s\n",
 426                  __func__, enable ? "enabled" : "disabled");
 427
 428         return count;
 429 }
 430 define_one_global_rw(boost);
 431
 432 static struct cpufreq_governor *__find_governor(const char *str_governor)
 433 {
 434         struct cpufreq_governor *t;
 435
 436         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 437                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 438                         return t;
 439
 440         return NULL;
 441 }
 442
 443 /**
 444  * cpufreq_parse_governor - parse a governor string
 445  */
 446 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 447                                 struct cpufreq_governor **governor)
 448 {
 449         int err = -EINVAL;
 450
 451         if (!cpufreq_driver)
 452                 goto out;
 453
 454         if (cpufreq_driver->setpolicy) {
 455                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 456                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 457                         err = 0;
 458                 } else if (!strnicmp(str_governor, "powersave",
 459                                                 CPUFREQ_NAME_LEN)) {
 460                         *policy = CPUFREQ_POLICY_POWERSAVE;
 461                         err = 0;
 462                 }
 463         } else if (has_target()) {
 464                 struct cpufreq_governor *t;
 465
 466                 mutex_lock(&cpufreq_governor_mutex);
 467
 468                 t = __find_governor(str_governor);
 469
 470                 if (t == NULL) {
 471                         int ret;
 472
 473                         mutex_unlock(&cpufreq_governor_mutex);
 474                         ret = request_module("cpufreq_%s", str_governor);
 475                         mutex_lock(&cpufreq_governor_mutex);
 476
 477                         if (ret == 0)
 478                                 t = __find_governor(str_governor);
 479                 }
 480
 481                 if (t != NULL) {
 482                         *governor = t;
 483                         err = 0;
 484                 }
 485
 486                 mutex_unlock(&cpufreq_governor_mutex);
 487         }
 488 out:
 489         return err;
 490 }
 491
 492 /**
 493  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 494  * print out cpufreq information
 495  *
 496  * Write out information from cpufreq_driver->policy[cpu]; object must be
 497  * "unsigned int".
 498  */
 499
 500 #define show_one(file_name, object)                     \
 501 static ssize_t show_##file_name                         \
 502 (struct cpufreq_policy *policy, char *buf)              \
 503 {                                                       \
 504         return sprintf(buf, "%u\n", policy->object);    \
 505 }
 506
 507 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 508 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 509 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 510 show_one(scaling_min_freq, min);
 511 show_one(scaling_max_freq, max);
 512 show_one(scaling_cur_freq, cur);
 513
 514 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 515                                 struct cpufreq_policy *new_policy);
 516
 517 /**
 518  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 519  */
 520 #define store_one(file_name, object)                    \
 521 static ssize_t store_##file_name                                        \
 522 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 523 {                                                                       \
 524         int ret;                                                        \
 525         struct cpufreq_policy new_policy;                               \
 526                                                                         \
 527         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 528         if (ret)                                                        \
 529                 return -EINVAL;                                         \
 530                                                                         \
 531         ret = sscanf(buf, "%u", &new_policy.object);                    \
 532         if (ret != 1)                                                   \
 533                 return -EINVAL;                                         \
 534                                                                         \
 535         ret = cpufreq_set_policy(policy, &new_policy);          \
 536         policy->user_policy.object = policy->object;                    \
 537                                                                         \
 538         return ret ? ret : count;                                       \
 539 }
 540
 541 store_one(scaling_min_freq, min);
 542 store_one(scaling_max_freq, max);
 543
 544 /**
 545  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 546  */
 547 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 548                                         char *buf)
 549 {
 550         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 551         if (!cur_freq)
 552                 return sprintf(buf, "<unknown>");
 553         return sprintf(buf, "%u\n", cur_freq);
 554 }
 555
 556 /**
 557  * show_scaling_governor - show the current policy for the specified CPU
 558  */
 559 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 560 {
 561         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 562                 return sprintf(buf, "powersave\n");
 563         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 564                 return sprintf(buf, "performance\n");
 565         else if (policy->governor)
 566                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 567                                 policy->governor->name);
 568         return -EINVAL;
 569 }
 570
 571 /**
 572  * store_scaling_governor - store policy for the specified CPU
 573  */
 574 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 575                                         const char *buf, size_t count)
 576 {
 577         int ret;
 578         char    str_governor[16];
 579         struct cpufreq_policy new_policy;
 580
 581         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 582         if (ret)
 583                 return ret;
 584
 585         ret = sscanf(buf, "%15s", str_governor);
 586         if (ret != 1)
 587                 return -EINVAL;
 588
 589         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 590                                                 &new_policy.governor))
 591                 return -EINVAL;
 592
 593         ret = cpufreq_set_policy(policy, &new_policy);
 594
 595         policy->user_policy.policy = policy->policy;
 596         policy->user_policy.governor = policy->governor;
 597
 598         if (ret)
 599                 return ret;
 600         else
 601                 return count;
 602 }
 603
 604 /**
 605  * show_scaling_driver - show the cpufreq driver currently loaded
 606  */
 607 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 608 {
 609         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 610 }
 611
 612 /**
 613  * show_scaling_available_governors - show the available CPUfreq governors
 614  */
 615 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 616                                                 char *buf)
 617 {
 618         ssize_t i = 0;
 619         struct cpufreq_governor *t;
 620
 621         if (!has_target()) {
 622                 i += sprintf(buf, "performance powersave");
 623                 goto out;
 624         }
 625
 626         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 627                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 628                     - (CPUFREQ_NAME_LEN + 2)))
 629                         goto out;
 630                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 631         }
 632 out:
 633         i += sprintf(&buf[i], "\n");
 634         return i;
 635 }
 636
 637 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 638 {
 639         ssize_t i = 0;
 640         unsigned int cpu;
 641
 642         for_each_cpu(cpu, mask) {
 643                 if (i)
 644                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 645                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 646                 if (i >= (PAGE_SIZE - 5))
 647                         break;
 648         }
 649         i += sprintf(&buf[i], "\n");
 650         return i;
 651 }
 652 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 653
 654 /**
 655  * show_related_cpus - show the CPUs affected by each transition even if
 656  * hw coordination is in use
 657  */
 658 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 659 {
 660         return cpufreq_show_cpus(policy->related_cpus, buf);
 661 }
 662
 663 /**
 664  * show_affected_cpus - show the CPUs affected by each transition
 665  */
 666 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 667 {
 668         return cpufreq_show_cpus(policy->cpus, buf);
 669 }
 670
 671 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 672                                         const char *buf, size_t count)
 673 {
 674         unsigned int freq = 0;
 675         unsigned int ret;
 676
 677         if (!policy->governor || !policy->governor->store_setspeed)
 678                 return -EINVAL;
 679
 680         ret = sscanf(buf, "%u", &freq);
 681         if (ret != 1)
 682                 return -EINVAL;
 683
 684         policy->governor->store_setspeed(policy, freq);
 685
 686         return count;
 687 }
 688
 689 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 690 {
 691         if (!policy->governor || !policy->governor->show_setspeed)
 692                 return sprintf(buf, "<unsupported>\n");
 693
 694         return policy->governor->show_setspeed(policy, buf);
 695 }
 696
 697 /**
 698  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 699  */
 700 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 701 {
 702         unsigned int limit;
 703         int ret;
 704         if (cpufreq_driver->bios_limit) {
 705                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 706                 if (!ret)
 707                         return sprintf(buf, "%u\n", limit);
 708         }
 709         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 710 }
 711
 712 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 713 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 714 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 715 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 716 cpufreq_freq_attr_ro(scaling_available_governors);
 717 cpufreq_freq_attr_ro(scaling_driver);
 718 cpufreq_freq_attr_ro(scaling_cur_freq);
 719 cpufreq_freq_attr_ro(bios_limit);
 720 cpufreq_freq_attr_ro(related_cpus);
 721 cpufreq_freq_attr_ro(affected_cpus);
 722 cpufreq_freq_attr_rw(scaling_min_freq);
 723 cpufreq_freq_attr_rw(scaling_max_freq);
 724 cpufreq_freq_attr_rw(scaling_governor);
 725 cpufreq_freq_attr_rw(scaling_setspeed);
 726
 727 static struct attribute *default_attrs[] = {
 728         &cpuinfo_min_freq.attr,
 729         &cpuinfo_max_freq.attr,
 730         &cpuinfo_transition_latency.attr,
 731         &scaling_min_freq.attr,
 732         &scaling_max_freq.attr,
 733         &affected_cpus.attr,
 734         &related_cpus.attr,
 735         &scaling_governor.attr,
 736         &scaling_driver.attr,
 737         &scaling_available_governors.attr,
 738         &scaling_setspeed.attr,
 739         NULL
 740 };
 741
 742 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 743 #define to_attr(a) container_of(a, struct freq_attr, attr)
 744
 745 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 746 {
 747         struct cpufreq_policy *policy = to_policy(kobj);
 748         struct freq_attr *fattr = to_attr(attr);
 749         ssize_t ret;
 750
 751         if (!down_read_trylock(&cpufreq_rwsem))
 752                 return -EINVAL;
 753
 754         down_read(&policy->rwsem);
 755
 756         if (fattr->show)
 757                 ret = fattr->show(policy, buf);
 758         else
 759                 ret = -EIO;
 760
 761         up_read(&policy->rwsem);
 762         up_read(&cpufreq_rwsem);
 763
 764         return ret;
 765 }
 766
 767 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 768                      const char *buf, size_t count)
 769 {
 770         struct cpufreq_policy *policy = to_policy(kobj);
 771         struct freq_attr *fattr = to_attr(attr);
 772         ssize_t ret = -EINVAL;
 773
 774         get_online_cpus();
 775
 776         if (!cpu_online(policy->cpu))
 777                 goto unlock;
 778
 779         if (!down_read_trylock(&cpufreq_rwsem))
 780                 goto unlock;
 781
 782         down_write(&policy->rwsem);
 783
 784         if (fattr->store)
 785                 ret = fattr->store(policy, buf, count);
 786         else
 787                 ret = -EIO;
 788
 789         up_write(&policy->rwsem);
 790
 791         up_read(&cpufreq_rwsem);
 792 unlock:
 793         put_online_cpus();
 794
 795         return ret;
 796 }
 797
 798 static void cpufreq_sysfs_release(struct kobject *kobj)
 799 {
 800         struct cpufreq_policy *policy = to_policy(kobj);
 801         pr_debug("last reference is dropped\n");
 802         complete(&policy->kobj_unregister);
 803 }
 804
 805 static const struct sysfs_ops sysfs_ops = {
 806         .show   = show,
 807         .store  = store,
 808 };
 809
 810 static struct kobj_type ktype_cpufreq = {
 811         .sysfs_ops      = &sysfs_ops,
 812         .default_attrs  = default_attrs,
 813         .release        = cpufreq_sysfs_release,
 814 };
 815
 816 struct kobject *cpufreq_global_kobject;
 817 EXPORT_SYMBOL(cpufreq_global_kobject);
 818
 819 static int cpufreq_global_kobject_usage;
 820
 821 int cpufreq_get_global_kobject(void)
 822 {
 823         if (!cpufreq_global_kobject_usage++)
 824                 return kobject_add(cpufreq_global_kobject,
 825                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 826
 827         return 0;
 828 }
 829 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 830
 831 void cpufreq_put_global_kobject(void)
 832 {
 833         if (!--cpufreq_global_kobject_usage)
 834                 kobject_del(cpufreq_global_kobject);
 835 }
 836 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 837
 838 int cpufreq_sysfs_create_file(const struct attribute *attr)
 839 {
 840         int ret = cpufreq_get_global_kobject();
 841
 842         if (!ret) {
 843                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 844                 if (ret)
 845                         cpufreq_put_global_kobject();
 846         }
 847
 848         return ret;
 849 }
 850 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 851
 852 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 853 {
 854         sysfs_remove_file(cpufreq_global_kobject, attr);
 855         cpufreq_put_global_kobject();
 856 }
 857 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 858
 859 /* symlink affected CPUs */
 860 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 861 {
 862         unsigned int j;
 863         int ret = 0;
 864
 865         for_each_cpu(j, policy->cpus) {
 866                 struct device *cpu_dev;
 867
 868                 if (j == policy->cpu)
 869                         continue;
 870
 871                 pr_debug("Adding link for CPU: %u\n", j);
 872                 cpu_dev = get_cpu_device(j);
 873                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 874                                         "cpufreq");
 875                 if (ret)
 876                         break;
 877         }
 878         return ret;
 879 }
 880
 881 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 882                                      struct device *dev)
 883 {
 884         struct freq_attr **drv_attr;
 885         int ret = 0;
 886
 887         /* prepare interface data */
 888         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 889                                    &dev->kobj, "cpufreq");
 890         if (ret)
 891                 return ret;
 892
 893         /* set up files for this cpu device */
 894         drv_attr = cpufreq_driver->attr;
 895         while ((drv_attr) && (*drv_attr)) {
 896                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 897                 if (ret)
 898                         goto err_out_kobj_put;
 899                 drv_attr++;
 900         }
 901         if (cpufreq_driver->get) {
 902                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 903                 if (ret)
 904                         goto err_out_kobj_put;
 905         }
 906         if (has_target()) {
 907                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 908                 if (ret)
 909                         goto err_out_kobj_put;
 910         }
 911         if (cpufreq_driver->bios_limit) {
 912                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 913                 if (ret)
 914                         goto err_out_kobj_put;
 915         }
 916
 917         ret = cpufreq_add_dev_symlink(policy);
 918         if (ret)
 919                 goto err_out_kobj_put;
 920
 921         return ret;
 922
 923 err_out_kobj_put:
 924         kobject_put(&policy->kobj);
 925         wait_for_completion(&policy->kobj_unregister);
 926         return ret;
 927 }
 928
 929 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 930 {
 931         struct cpufreq_governor *gov = NULL;
 932         struct cpufreq_policy new_policy;
 933         int ret = 0;
 934
 935         memcpy(&new_policy, policy, sizeof(*policy));
 936
 937         /* Update governor of new_policy to the governor used before hotplug */
 938         gov = __find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
 939         if (gov)
 940                 pr_debug("Restoring governor %s for cpu %d\n",
 941                                 policy->governor->name, policy->cpu);
 942         else
 943                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 944
 945         new_policy.governor = gov;
 946
 947         /* Use the default policy if its valid. */
 948         if (cpufreq_driver->setpolicy)
 949                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 950
 951         /* set default policy */
 952         ret = cpufreq_set_policy(policy, &new_policy);
 953         if (ret) {
 954                 pr_debug("setting policy failed\n");
 955                 if (cpufreq_driver->exit)
 956                         cpufreq_driver->exit(policy);
 957         }
 958 }
 959
 960 #ifdef CONFIG_HOTPLUG_CPU
 961 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 962                                   unsigned int cpu, struct device *dev)
 963 {
 964         int ret = 0;
 965         unsigned long flags;
 966
 967         if (has_target()) {
 968                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 969                 if (ret) {
 970                         pr_err("%s: Failed to stop governor\n", __func__);
 971                         return ret;
 972                 }
 973         }
 974
 975         down_write(&policy->rwsem);
 976
 977         write_lock_irqsave(&cpufreq_driver_lock, flags);
 978
 979         cpumask_set_cpu(cpu, policy->cpus);
 980         per_cpu(cpufreq_cpu_data, cpu) = policy;
 981         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 982
 983         up_write(&policy->rwsem);
 984
 985         if (has_target()) {
 986                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
 987                 if (!ret)
 988                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 989
 990                 if (ret) {
 991                         pr_err("%s: Failed to start governor\n", __func__);
 992                         return ret;
 993                 }
 994         }
 995
 996         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 997 }
 998 #endif
 999
1000 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1001 {
1002         struct cpufreq_policy *policy;
1003         unsigned long flags;
1004
1005         read_lock_irqsave(&cpufreq_driver_lock, flags);
1006
1007         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
1008
1009         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1010
1011         policy->governor = NULL;
1012
1013         return policy;
1014 }
1015
1016 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1017 {
1018         struct cpufreq_policy *policy;
1019
1020         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1021         if (!policy)
1022                 return NULL;
1023
1024         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1025                 goto err_free_policy;
1026
1027         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1028                 goto err_free_cpumask;
1029
1030         INIT_LIST_HEAD(&policy->policy_list);
1031         init_rwsem(&policy->rwsem);
1032         spin_lock_init(&policy->transition_lock);
1033         init_waitqueue_head(&policy->transition_wait);
1034
1035         return policy;
1036
1037 err_free_cpumask:
1038         free_cpumask_var(policy->cpus);
1039 err_free_policy:
1040         kfree(policy);
1041
1042         return NULL;
1043 }
1044
1045 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1046 {
1047         struct kobject *kobj;
1048         struct completion *cmp;
1049
1050         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1051                         CPUFREQ_REMOVE_POLICY, policy);
1052
1053         down_read(&policy->rwsem);
1054         kobj = &policy->kobj;
1055         cmp = &policy->kobj_unregister;
1056         up_read(&policy->rwsem);
1057         kobject_put(kobj);
1058
1059         /*
1060          * We need to make sure that the underlying kobj is
1061          * actually not referenced anymore by anybody before we
1062          * proceed with unloading.
1063          */
1064         pr_debug("waiting for dropping of refcount\n");
1065         wait_for_completion(cmp);
1066         pr_debug("wait complete\n");
1067 }
1068
1069 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1070 {
1071         free_cpumask_var(policy->related_cpus);
1072         free_cpumask_var(policy->cpus);
1073         kfree(policy);
1074 }
1075
1076 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1077 {
1078         if (WARN_ON(cpu == policy->cpu))
1079                 return;
1080
1081         down_write(&policy->rwsem);
1082
1083         policy->last_cpu = policy->cpu;
1084         policy->cpu = cpu;
1085
1086         up_write(&policy->rwsem);
1087
1088         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1089                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1090 }
1091
1092 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1093 {
1094         unsigned int j, cpu = dev->id;
1095         int ret = -ENOMEM;
1096         struct cpufreq_policy *policy;
1097         unsigned long flags;
1098         bool recover_policy = cpufreq_suspended;
1099 #ifdef CONFIG_HOTPLUG_CPU
1100         struct cpufreq_policy *tpolicy;
1101 #endif
1102
1103         if (cpu_is_offline(cpu))
1104                 return 0;
1105
1106         pr_debug("adding CPU %u\n", cpu);
1107
1108 #ifdef CONFIG_SMP
1109         /* check whether a different CPU already registered this
1110          * CPU because it is in the same boat. */
1111         policy = cpufreq_cpu_get(cpu);
1112         if (unlikely(policy)) {
1113                 cpufreq_cpu_put(policy);
1114                 return 0;
1115         }
1116 #endif
1117
1118         if (!down_read_trylock(&cpufreq_rwsem))
1119                 return 0;
1120
1121 #ifdef CONFIG_HOTPLUG_CPU
1122         /* Check if this cpu was hot-unplugged earlier and has siblings */
1123         read_lock_irqsave(&cpufreq_driver_lock, flags);
1124         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1125                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1126                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1127                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1128                         up_read(&cpufreq_rwsem);
1129                         return ret;
1130                 }
1131         }
1132         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1133 #endif
1134
1135         /*
1136          * Restore the saved policy when doing light-weight init and fall back
1137          * to the full init if that fails.
1138          */
1139         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1140         if (!policy) {
1141                 recover_policy = false;
1142                 policy = cpufreq_policy_alloc();
1143                 if (!policy)
1144                         goto nomem_out;
1145         }
1146
1147         /*
1148          * In the resume path, since we restore a saved policy, the assignment
1149          * to policy->cpu is like an update of the existing policy, rather than
1150          * the creation of a brand new one. So we need to perform this update
1151          * by invoking update_policy_cpu().
1152          */
1153         if (recover_policy && cpu != policy->cpu)
1154                 update_policy_cpu(policy, cpu);
1155         else
1156                 policy->cpu = cpu;
1157
1158         cpumask_copy(policy->cpus, cpumask_of(cpu));
1159
1160         init_completion(&policy->kobj_unregister);
1161         INIT_WORK(&policy->update, handle_update);
1162
1163         /* call driver. From then on the cpufreq must be able
1164          * to accept all calls to ->verify and ->setpolicy for this CPU
1165          */
1166         ret = cpufreq_driver->init(policy);
1167         if (ret) {
1168                 pr_debug("initialization failed\n");
1169                 goto err_set_policy_cpu;
1170         }
1171
1172         /* related cpus should atleast have policy->cpus */
1173         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1174
1175         /*
1176          * affected cpus must always be the one, which are online. We aren't
1177          * managing offline cpus here.
1178          */
1179         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1180
1181         if (!recover_policy) {
1182                 policy->user_policy.min = policy->min;
1183                 policy->user_policy.max = policy->max;
1184         }
1185
1186         down_write(&policy->rwsem);
1187         write_lock_irqsave(&cpufreq_driver_lock, flags);
1188         for_each_cpu(j, policy->cpus)
1189                 per_cpu(cpufreq_cpu_data, j) = policy;
1190         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1191
1192         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1193                 policy->cur = cpufreq_driver->get(policy->cpu);
1194                 if (!policy->cur) {
1195                         pr_err("%s: ->get() failed\n", __func__);
1196                         goto err_get_freq;
1197                 }
1198         }
1199
1200         /*
1201          * Sometimes boot loaders set CPU frequency to a value outside of
1202          * frequency table present with cpufreq core. In such cases CPU might be
1203          * unstable if it has to run on that frequency for long duration of time
1204          * and so its better to set it to a frequency which is specified in
1205          * freq-table. This also makes cpufreq stats inconsistent as
1206          * cpufreq-stats would fail to register because current frequency of CPU
1207          * isn't found in freq-table.
1208          *
1209          * Because we don't want this change to effect boot process badly, we go
1210          * for the next freq which is >= policy->cur ('cur' must be set by now,
1211          * otherwise we will end up setting freq to lowest of the table as 'cur'
1212          * is initialized to zero).
1213          *
1214          * We are passing target-freq as "policy->cur - 1" otherwise
1215          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1216          * equal to target-freq.
1217          */
1218         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1219             && has_target()) {
1220                 /* Are we running at unknown frequency ? */
1221                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1222                 if (ret == -EINVAL) {
1223                         /* Warn user and fix it */
1224                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1225                                 __func__, policy->cpu, policy->cur);
1226                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1227                                 CPUFREQ_RELATION_L);
1228
1229                         /*
1230                          * Reaching here after boot in a few seconds may not
1231                          * mean that system will remain stable at "unknown"
1232                          * frequency for longer duration. Hence, a BUG_ON().
1233                          */
1234                         BUG_ON(ret);
1235                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1236                                 __func__, policy->cpu, policy->cur);
1237                 }
1238         }
1239
1240         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1241                                      CPUFREQ_START, policy);
1242
1243         if (!recover_policy) {
1244                 ret = cpufreq_add_dev_interface(policy, dev);
1245                 if (ret)
1246                         goto err_out_unregister;
1247                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1248                                 CPUFREQ_CREATE_POLICY, policy);
1249         }
1250
1251         write_lock_irqsave(&cpufreq_driver_lock, flags);
1252         list_add(&policy->policy_list, &cpufreq_policy_list);
1253         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1254
1255         cpufreq_init_policy(policy);
1256
1257         if (!recover_policy) {
1258                 policy->user_policy.policy = policy->policy;
1259                 policy->user_policy.governor = policy->governor;
1260         }
1261         up_write(&policy->rwsem);
1262
1263         kobject_uevent(&policy->kobj, KOBJ_ADD);
1264         up_read(&cpufreq_rwsem);
1265
1266         pr_debug("initialization complete\n");
1267
1268         return 0;
1269
1270 err_out_unregister:
1271 err_get_freq:
1272         write_lock_irqsave(&cpufreq_driver_lock, flags);
1273         for_each_cpu(j, policy->cpus)
1274                 per_cpu(cpufreq_cpu_data, j) = NULL;
1275         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1276
1277         if (cpufreq_driver->exit)
1278                 cpufreq_driver->exit(policy);
1279 err_set_policy_cpu:
1280         if (recover_policy) {
1281                 /* Do not leave stale fallback data behind. */
1282                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1283                 cpufreq_policy_put_kobj(policy);
1284         }
1285         cpufreq_policy_free(policy);
1286
1287 nomem_out:
1288         up_read(&cpufreq_rwsem);
1289
1290         return ret;
1291 }
1292
1293 /**
1294  * cpufreq_add_dev - add a CPU device
1295  *
1296  * Adds the cpufreq interface for a CPU device.
1297  *
1298  * The Oracle says: try running cpufreq registration/unregistration concurrently
1299  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1300  * mess up, but more thorough testing is needed. - Mathieu
1301  */
1302 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1303 {
1304         return __cpufreq_add_dev(dev, sif);
1305 }
1306
1307 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1308                                            unsigned int old_cpu)
1309 {
1310         struct device *cpu_dev;
1311         int ret;
1312
1313         /* first sibling now owns the new sysfs dir */
1314         cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1315
1316         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1317         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1318         if (ret) {
1319                 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1320
1321                 down_write(&policy->rwsem);
1322                 cpumask_set_cpu(old_cpu, policy->cpus);
1323                 up_write(&policy->rwsem);
1324
1325                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1326                                         "cpufreq");
1327
1328                 return -EINVAL;
1329         }
1330
1331         return cpu_dev->id;
1332 }
1333
1334 static int __cpufreq_remove_dev_prepare(struct device *dev,
1335                                         struct subsys_interface *sif)
1336 {
1337         unsigned int cpu = dev->id, cpus;
1338         int new_cpu, ret;
1339         unsigned long flags;
1340         struct cpufreq_policy *policy;
1341
1342         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1343
1344         write_lock_irqsave(&cpufreq_driver_lock, flags);
1345
1346         policy = per_cpu(cpufreq_cpu_data, cpu);
1347
1348         /* Save the policy somewhere when doing a light-weight tear-down */
1349         if (cpufreq_suspended)
1350                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1351
1352         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1353
1354         if (!policy) {
1355                 pr_debug("%s: No cpu_data found\n", __func__);
1356                 return -EINVAL;
1357         }
1358
1359         if (has_target()) {
1360                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1361                 if (ret) {
1362                         pr_err("%s: Failed to stop governor\n", __func__);
1363                         return ret;
1364                 }
1365         }
1366
1367         if (!cpufreq_driver->setpolicy)
1368                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1369                         policy->governor->name, CPUFREQ_NAME_LEN);
1370
1371         down_read(&policy->rwsem);
1372         cpus = cpumask_weight(policy->cpus);
1373         up_read(&policy->rwsem);
1374
1375         if (cpu != policy->cpu) {
1376                 sysfs_remove_link(&dev->kobj, "cpufreq");
1377         } else if (cpus > 1) {
1378                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1379                 if (new_cpu >= 0) {
1380                         update_policy_cpu(policy, new_cpu);
1381
1382                         if (!cpufreq_suspended)
1383                                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1384                                          __func__, new_cpu, cpu);
1385                 }
1386         } else if (cpufreq_driver->stop_cpu && cpufreq_driver->setpolicy) {
1387                 cpufreq_driver->stop_cpu(policy);
1388         }
1389
1390         return 0;
1391 }
1392
1393 static int __cpufreq_remove_dev_finish(struct device *dev,
1394                                        struct subsys_interface *sif)
1395 {
1396         unsigned int cpu = dev->id, cpus;
1397         int ret;
1398         unsigned long flags;
1399         struct cpufreq_policy *policy;
1400
1401         read_lock_irqsave(&cpufreq_driver_lock, flags);
1402         policy = per_cpu(cpufreq_cpu_data, cpu);
1403         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1404
1405         if (!policy) {
1406                 pr_debug("%s: No cpu_data found\n", __func__);
1407                 return -EINVAL;
1408         }
1409
1410         down_write(&policy->rwsem);
1411         cpus = cpumask_weight(policy->cpus);
1412
1413         if (cpus > 1)
1414                 cpumask_clear_cpu(cpu, policy->cpus);
1415         up_write(&policy->rwsem);
1416
1417         /* If cpu is last user of policy, free policy */
1418         if (cpus == 1) {
1419                 if (has_target()) {
1420                         ret = __cpufreq_governor(policy,
1421                                         CPUFREQ_GOV_POLICY_EXIT);
1422                         if (ret) {
1423                                 pr_err("%s: Failed to exit governor\n",
1424                                        __func__);
1425                                 return ret;
1426                         }
1427                 }
1428
1429                 if (!cpufreq_suspended)
1430                         cpufreq_policy_put_kobj(policy);
1431
1432                 /*
1433                  * Perform the ->exit() even during light-weight tear-down,
1434                  * since this is a core component, and is essential for the
1435                  * subsequent light-weight ->init() to succeed.
1436                  */
1437                 if (cpufreq_driver->exit)
1438                         cpufreq_driver->exit(policy);
1439
1440                 /* Remove policy from list of active policies */
1441                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1442                 list_del(&policy->policy_list);
1443                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1444
1445                 if (!cpufreq_suspended)
1446                         cpufreq_policy_free(policy);
1447         } else if (has_target()) {
1448                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1449                 if (!ret)
1450                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1451
1452                 if (ret) {
1453                         pr_err("%s: Failed to start governor\n", __func__);
1454                         return ret;
1455                 }
1456         }
1457
1458         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1459         return 0;
1460 }
1461
1462 /**
1463  * cpufreq_remove_dev - remove a CPU device
1464  *
1465  * Removes the cpufreq interface for a CPU device.
1466  */
1467 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1468 {
1469         unsigned int cpu = dev->id;
1470         int ret;
1471
1472         if (cpu_is_offline(cpu))
1473                 return 0;
1474
1475         ret = __cpufreq_remove_dev_prepare(dev, sif);
1476
1477         if (!ret)
1478                 ret = __cpufreq_remove_dev_finish(dev, sif);
1479
1480         return ret;
1481 }
1482
1483 static void handle_update(struct work_struct *work)
1484 {
1485         struct cpufreq_policy *policy =
1486                 container_of(work, struct cpufreq_policy, update);
1487         unsigned int cpu = policy->cpu;
1488         pr_debug("handle_update for cpu %u called\n", cpu);
1489         cpufreq_update_policy(cpu);
1490 }
1491
1492 /**
1493  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1494  *      in deep trouble.
1495  *      @cpu: cpu number
1496  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1497  *      @new_freq: CPU frequency the CPU actually runs at
1498  *
1499  *      We adjust to current frequency first, and need to clean up later.
1500  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1501  */
1502 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1503                                 unsigned int new_freq)
1504 {
1505         struct cpufreq_policy *policy;
1506         struct cpufreq_freqs freqs;
1507         unsigned long flags;
1508
1509         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1510                  old_freq, new_freq);
1511
1512         freqs.old = old_freq;
1513         freqs.new = new_freq;
1514
1515         read_lock_irqsave(&cpufreq_driver_lock, flags);
1516         policy = per_cpu(cpufreq_cpu_data, cpu);
1517         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1518
1519         cpufreq_freq_transition_begin(policy, &freqs);
1520         cpufreq_freq_transition_end(policy, &freqs, 0);
1521 }
1522
1523 /**
1524  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1525  * @cpu: CPU number
1526  *
1527  * This is the last known freq, without actually getting it from the driver.
1528  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1529  */
1530 unsigned int cpufreq_quick_get(unsigned int cpu)
1531 {
1532         struct cpufreq_policy *policy;
1533         unsigned int ret_freq = 0;
1534
1535         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1536                 return cpufreq_driver->get(cpu);
1537
1538         policy = cpufreq_cpu_get(cpu);
1539         if (policy) {
1540                 ret_freq = policy->cur;
1541                 cpufreq_cpu_put(policy);
1542         }
1543
1544         return ret_freq;
1545 }
1546 EXPORT_SYMBOL(cpufreq_quick_get);
1547
1548 /**
1549  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1550  * @cpu: CPU number
1551  *
1552  * Just return the max possible frequency for a given CPU.
1553  */
1554 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1555 {
1556         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1557         unsigned int ret_freq = 0;
1558
1559         if (policy) {
1560                 ret_freq = policy->max;
1561                 cpufreq_cpu_put(policy);
1562         }
1563
1564         return ret_freq;
1565 }
1566 EXPORT_SYMBOL(cpufreq_quick_get_max);
1567
1568 static unsigned int __cpufreq_get(unsigned int cpu)
1569 {
1570         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1571         unsigned int ret_freq = 0;
1572
1573         if (!cpufreq_driver->get)
1574                 return ret_freq;
1575
1576         ret_freq = cpufreq_driver->get(cpu);
1577
1578         if (ret_freq && policy->cur &&
1579                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1580                 /* verify no discrepancy between actual and
1581                                         saved value exists */
1582                 if (unlikely(ret_freq != policy->cur)) {
1583                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1584                         schedule_work(&policy->update);
1585                 }
1586         }
1587
1588         return ret_freq;
1589 }
1590
1591 /**
1592  * cpufreq_get - get the current CPU frequency (in kHz)
1593  * @cpu: CPU number
1594  *
1595  * Get the CPU current (static) CPU frequency
1596  */
1597 unsigned int cpufreq_get(unsigned int cpu)
1598 {
1599         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1600         unsigned int ret_freq = 0;
1601
1602         if (policy) {
1603                 down_read(&policy->rwsem);
1604                 ret_freq = __cpufreq_get(cpu);
1605                 up_read(&policy->rwsem);
1606
1607                 cpufreq_cpu_put(policy);
1608         }
1609
1610         return ret_freq;
1611 }
1612 EXPORT_SYMBOL(cpufreq_get);
1613
1614 static struct subsys_interface cpufreq_interface = {
1615         .name           = "cpufreq",
1616         .subsys         = &cpu_subsys,
1617         .add_dev        = cpufreq_add_dev,
1618         .remove_dev     = cpufreq_remove_dev,
1619 };
1620
1621 /*
1622  * In case platform wants some specific frequency to be configured
1623  * during suspend..
1624  */
1625 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1626 {
1627         int ret;
1628
1629         if (!policy->suspend_freq) {
1630                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1631                 return -EINVAL;
1632         }
1633
1634         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1635                         policy->suspend_freq);
1636
1637         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1638                         CPUFREQ_RELATION_H);
1639         if (ret)
1640                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1641                                 __func__, policy->suspend_freq, ret);
1642
1643         return ret;
1644 }
1645 EXPORT_SYMBOL(cpufreq_generic_suspend);
1646
1647 /**
1648  * cpufreq_suspend() - Suspend CPUFreq governors
1649  *
1650  * Called during system wide Suspend/Hibernate cycles for suspending governors
1651  * as some platforms can't change frequency after this point in suspend cycle.
1652  * Because some of the devices (like: i2c, regulators, etc) they use for
1653  * changing frequency are suspended quickly after this point.
1654  */
1655 void cpufreq_suspend(void)
1656 {
1657         struct cpufreq_policy *policy;
1658
1659         if (!cpufreq_driver)
1660                 return;
1661
1662         if (!has_target())
1663                 return;
1664
1665         pr_debug("%s: Suspending Governors\n", __func__);
1666
1667         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1668                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1669                         pr_err("%s: Failed to stop governor for policy: %p\n",
1670                                 __func__, policy);
1671                 else if (cpufreq_driver->suspend
1672                     && cpufreq_driver->suspend(policy))
1673                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1674                                 policy);
1675         }
1676
1677         cpufreq_suspended = true;
1678 }
1679
1680 /**
1681  * cpufreq_resume() - Resume CPUFreq governors
1682  *
1683  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1684  * are suspended with cpufreq_suspend().
1685  */
1686 void cpufreq_resume(void)
1687 {
1688         struct cpufreq_policy *policy;
1689
1690         if (!cpufreq_driver)
1691                 return;
1692
1693         if (!has_target())
1694                 return;
1695
1696         pr_debug("%s: Resuming Governors\n", __func__);
1697
1698         cpufreq_suspended = false;
1699
1700         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1701                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1702                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1703                                 policy);
1704                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1705                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1706                         pr_err("%s: Failed to start governor for policy: %p\n",
1707                                 __func__, policy);
1708
1709                 /*
1710                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1711                  * policy in list. It will verify that the current freq is in
1712                  * sync with what we believe it to be.
1713                  */
1714                 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1715                         schedule_work(&policy->update);
1716         }
1717 }
1718
1719 /**
1720  *      cpufreq_get_current_driver - return current driver's name
1721  *
1722  *      Return the name string of the currently loaded cpufreq driver
1723  *      or NULL, if none.
1724  */
1725 const char *cpufreq_get_current_driver(void)
1726 {
1727         if (cpufreq_driver)
1728                 return cpufreq_driver->name;
1729
1730         return NULL;
1731 }
1732 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1733
1734 /*********************************************************************
1735  *                     NOTIFIER LISTS INTERFACE                      *
1736  *********************************************************************/
1737
1738 /**
1739  *      cpufreq_register_notifier - register a driver with cpufreq
1740  *      @nb: notifier function to register
1741  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1742  *
1743  *      Add a driver to one of two lists: either a list of drivers that
1744  *      are notified about clock rate changes (once before and once after
1745  *      the transition), or a list of drivers that are notified about
1746  *      changes in cpufreq policy.
1747  *
1748  *      This function may sleep, and has the same return conditions as
1749  *      blocking_notifier_chain_register.
1750  */
1751 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1752 {
1753         int ret;
1754
1755         if (cpufreq_disabled())
1756                 return -EINVAL;
1757
1758         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1759
1760         switch (list) {
1761         case CPUFREQ_TRANSITION_NOTIFIER:
1762                 ret = srcu_notifier_chain_register(
1763                                 &cpufreq_transition_notifier_list, nb);
1764                 break;
1765         case CPUFREQ_POLICY_NOTIFIER:
1766                 ret = blocking_notifier_chain_register(
1767                                 &cpufreq_policy_notifier_list, nb);
1768                 break;
1769         default:
1770                 ret = -EINVAL;
1771         }
1772
1773         return ret;
1774 }
1775 EXPORT_SYMBOL(cpufreq_register_notifier);
1776
1777 /**
1778  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1779  *      @nb: notifier block to be unregistered
1780  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1781  *
1782  *      Remove a driver from the CPU frequency notifier list.
1783  *
1784  *      This function may sleep, and has the same return conditions as
1785  *      blocking_notifier_chain_unregister.
1786  */
1787 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1788 {
1789         int ret;
1790
1791         if (cpufreq_disabled())
1792                 return -EINVAL;
1793
1794         switch (list) {
1795         case CPUFREQ_TRANSITION_NOTIFIER:
1796                 ret = srcu_notifier_chain_unregister(
1797                                 &cpufreq_transition_notifier_list, nb);
1798                 break;
1799         case CPUFREQ_POLICY_NOTIFIER:
1800                 ret = blocking_notifier_chain_unregister(
1801                                 &cpufreq_policy_notifier_list, nb);
1802                 break;
1803         default:
1804                 ret = -EINVAL;
1805         }
1806
1807         return ret;
1808 }
1809 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1810
1811
1812 /*********************************************************************
1813  *                              GOVERNORS                            *
1814  *********************************************************************/
1815
1816 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1817                             unsigned int target_freq,
1818                             unsigned int relation)
1819 {
1820         int retval = -EINVAL;
1821         unsigned int old_target_freq = target_freq;
1822
1823         if (cpufreq_disabled())
1824                 return -ENODEV;
1825
1826         /* Make sure that target_freq is within supported range */
1827         if (target_freq > policy->max)
1828                 target_freq = policy->max;
1829         if (target_freq < policy->min)
1830                 target_freq = policy->min;
1831
1832         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1833                  policy->cpu, target_freq, relation, old_target_freq);
1834
1835         /*
1836          * This might look like a redundant call as we are checking it again
1837          * after finding index. But it is left intentionally for cases where
1838          * exactly same freq is called again and so we can save on few function
1839          * calls.
1840          */
1841         if (target_freq == policy->cur)
1842                 return 0;
1843
1844         if (cpufreq_driver->target)
1845                 retval = cpufreq_driver->target(policy, target_freq, relation);
1846         else if (cpufreq_driver->target_index) {
1847                 struct cpufreq_frequency_table *freq_table;
1848                 struct cpufreq_freqs freqs;
1849                 bool notify;
1850                 int index;
1851
1852                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1853                 if (unlikely(!freq_table)) {
1854                         pr_err("%s: Unable to find freq_table\n", __func__);
1855                         goto out;
1856                 }
1857
1858                 retval = cpufreq_frequency_table_target(policy, freq_table,
1859                                 target_freq, relation, &index);
1860                 if (unlikely(retval)) {
1861                         pr_err("%s: Unable to find matching freq\n", __func__);
1862                         goto out;
1863                 }
1864
1865                 if (freq_table[index].frequency == policy->cur) {
1866                         retval = 0;
1867                         goto out;
1868                 }
1869
1870                 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1871
1872                 if (notify) {
1873                         freqs.old = policy->cur;
1874                         freqs.new = freq_table[index].frequency;
1875                         freqs.flags = 0;
1876
1877                         pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1878                                  __func__, policy->cpu, freqs.old, freqs.new);
1879
1880                         cpufreq_freq_transition_begin(policy, &freqs);
1881                 }
1882
1883                 retval = cpufreq_driver->target_index(policy, index);
1884                 if (retval)
1885                         pr_err("%s: Failed to change cpu frequency: %d\n",
1886                                __func__, retval);
1887
1888                 if (notify)
1889                         cpufreq_freq_transition_end(policy, &freqs, retval);
1890         }
1891
1892 out:
1893         return retval;
1894 }
1895 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1896
1897 int cpufreq_driver_target(struct cpufreq_policy *policy,
1898                           unsigned int target_freq,
1899                           unsigned int relation)
1900 {
1901         int ret = -EINVAL;
1902
1903         down_write(&policy->rwsem);
1904
1905         ret = __cpufreq_driver_target(policy, target_freq, relation);
1906
1907         up_write(&policy->rwsem);
1908
1909         return ret;
1910 }
1911 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1912
1913 /*
1914  * when "event" is CPUFREQ_GOV_LIMITS
1915  */
1916
1917 static int __cpufreq_governor(struct cpufreq_policy *policy,
1918                                         unsigned int event)
1919 {
1920         int ret;
1921
1922         /* Only must be defined when default governor is known to have latency
1923            restrictions, like e.g. conservative or ondemand.
1924            That this is the case is already ensured in Kconfig
1925         */
1926 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1927         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1928 #else
1929         struct cpufreq_governor *gov = NULL;
1930 #endif
1931
1932         /* Don't start any governor operations if we are entering suspend */
1933         if (cpufreq_suspended)
1934                 return 0;
1935
1936         if (policy->governor->max_transition_latency &&
1937             policy->cpuinfo.transition_latency >
1938             policy->governor->max_transition_latency) {
1939                 if (!gov)
1940                         return -EINVAL;
1941                 else {
1942                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
1943                                 policy->governor->name, gov->name);
1944                         policy->governor = gov;
1945                 }
1946         }
1947
1948         if (event == CPUFREQ_GOV_POLICY_INIT)
1949                 if (!try_module_get(policy->governor->owner))
1950                         return -EINVAL;
1951
1952         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1953                  policy->cpu, event);
1954
1955         mutex_lock(&cpufreq_governor_lock);
1956         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1957             || (!policy->governor_enabled
1958             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1959                 mutex_unlock(&cpufreq_governor_lock);
1960                 return -EBUSY;
1961         }
1962
1963         if (event == CPUFREQ_GOV_STOP)
1964                 policy->governor_enabled = false;
1965         else if (event == CPUFREQ_GOV_START)
1966                 policy->governor_enabled = true;
1967
1968         mutex_unlock(&cpufreq_governor_lock);
1969
1970         ret = policy->governor->governor(policy, event);
1971
1972         if (!ret) {
1973                 if (event == CPUFREQ_GOV_POLICY_INIT)
1974                         policy->governor->initialized++;
1975                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1976                         policy->governor->initialized--;
1977         } else {
1978                 /* Restore original values */
1979                 mutex_lock(&cpufreq_governor_lock);
1980                 if (event == CPUFREQ_GOV_STOP)
1981                         policy->governor_enabled = true;
1982                 else if (event == CPUFREQ_GOV_START)
1983                         policy->governor_enabled = false;
1984                 mutex_unlock(&cpufreq_governor_lock);
1985         }
1986
1987         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1988                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1989                 module_put(policy->governor->owner);
1990
1991         return ret;
1992 }
1993
1994 int cpufreq_register_governor(struct cpufreq_governor *governor)
1995 {
1996         int err;
1997
1998         if (!governor)
1999                 return -EINVAL;
2000
2001         if (cpufreq_disabled())
2002                 return -ENODEV;
2003
2004         mutex_lock(&cpufreq_governor_mutex);
2005
2006         governor->initialized = 0;
2007         err = -EBUSY;
2008         if (__find_governor(governor->name) == NULL) {
2009                 err = 0;
2010                 list_add(&governor->governor_list, &cpufreq_governor_list);
2011         }
2012
2013         mutex_unlock(&cpufreq_governor_mutex);
2014         return err;
2015 }
2016 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2017
2018 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2019 {
2020         int cpu;
2021
2022         if (!governor)
2023                 return;
2024
2025         if (cpufreq_disabled())
2026                 return;
2027
2028         for_each_present_cpu(cpu) {
2029                 if (cpu_online(cpu))
2030                         continue;
2031                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2032                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2033         }
2034
2035         mutex_lock(&cpufreq_governor_mutex);
2036         list_del(&governor->governor_list);
2037         mutex_unlock(&cpufreq_governor_mutex);
2038         return;
2039 }
2040 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2041
2042
2043 /*********************************************************************
2044  *                          POLICY INTERFACE                         *
2045  *********************************************************************/
2046
2047 /**
2048  * cpufreq_get_policy - get the current cpufreq_policy
2049  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2050  *      is written
2051  *
2052  * Reads the current cpufreq policy.
2053  */
2054 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2055 {
2056         struct cpufreq_policy *cpu_policy;
2057         if (!policy)
2058                 return -EINVAL;
2059
2060         cpu_policy = cpufreq_cpu_get(cpu);
2061         if (!cpu_policy)
2062                 return -EINVAL;
2063
2064         memcpy(policy, cpu_policy, sizeof(*policy));
2065
2066         cpufreq_cpu_put(cpu_policy);
2067         return 0;
2068 }
2069 EXPORT_SYMBOL(cpufreq_get_policy);
2070
2071 /*
2072  * policy : current policy.
2073  * new_policy: policy to be set.
2074  */
2075 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2076                                 struct cpufreq_policy *new_policy)
2077 {
2078         struct cpufreq_governor *old_gov;
2079         int ret;
2080
2081         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2082                  new_policy->cpu, new_policy->min, new_policy->max);
2083
2084         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2085
2086         if (new_policy->min > policy->max || new_policy->max < policy->min)
2087                 return -EINVAL;
2088
2089         /* verify the cpu speed can be set within this limit */
2090         ret = cpufreq_driver->verify(new_policy);
2091         if (ret)
2092                 return ret;
2093
2094         /* adjust if necessary - all reasons */
2095         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2096                         CPUFREQ_ADJUST, new_policy);
2097
2098         /* adjust if necessary - hardware incompatibility*/
2099         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2100                         CPUFREQ_INCOMPATIBLE, new_policy);
2101
2102         /*
2103          * verify the cpu speed can be set within this limit, which might be
2104          * different to the first one
2105          */
2106         ret = cpufreq_driver->verify(new_policy);
2107         if (ret)
2108                 return ret;
2109
2110         /* notification of the new policy */
2111         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2112                         CPUFREQ_NOTIFY, new_policy);
2113
2114         policy->min = new_policy->min;
2115         policy->max = new_policy->max;
2116
2117         pr_debug("new min and max freqs are %u - %u kHz\n",
2118                  policy->min, policy->max);
2119
2120         if (cpufreq_driver->setpolicy) {
2121                 policy->policy = new_policy->policy;
2122                 pr_debug("setting range\n");
2123                 return cpufreq_driver->setpolicy(new_policy);
2124         }
2125
2126         if (new_policy->governor == policy->governor)
2127                 goto out;
2128
2129         pr_debug("governor switch\n");
2130
2131         /* save old, working values */
2132         old_gov = policy->governor;
2133         /* end old governor */
2134         if (old_gov) {
2135                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2136                 up_write(&policy->rwsem);
2137                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2138                 down_write(&policy->rwsem);
2139         }
2140
2141         /* start new governor */
2142         policy->governor = new_policy->governor;
2143         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2144                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2145                         goto out;
2146
2147                 up_write(&policy->rwsem);
2148                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2149                 down_write(&policy->rwsem);
2150         }
2151
2152         /* new governor failed, so re-start old one */
2153         pr_debug("starting governor %s failed\n", policy->governor->name);
2154         if (old_gov) {
2155                 policy->governor = old_gov;
2156                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2157                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2158         }
2159
2160         return -EINVAL;
2161
2162  out:
2163         pr_debug("governor: change or update limits\n");
2164         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2165 }
2166
2167 /**
2168  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2169  *      @cpu: CPU which shall be re-evaluated
2170  *
2171  *      Useful for policy notifiers which have different necessities
2172  *      at different times.
2173  */
2174 int cpufreq_update_policy(unsigned int cpu)
2175 {
2176         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2177         struct cpufreq_policy new_policy;
2178         int ret;
2179
2180         if (!policy) {
2181                 ret = -ENODEV;
2182                 goto no_policy;
2183         }
2184
2185         down_write(&policy->rwsem);
2186
2187         pr_debug("updating policy for CPU %u\n", cpu);
2188         memcpy(&new_policy, policy, sizeof(*policy));
2189         new_policy.min = policy->user_policy.min;
2190         new_policy.max = policy->user_policy.max;
2191         new_policy.policy = policy->user_policy.policy;
2192         new_policy.governor = policy->user_policy.governor;
2193
2194         /*
2195          * BIOS might change freq behind our back
2196          * -> ask driver for current freq and notify governors about a change
2197          */
2198         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2199                 new_policy.cur = cpufreq_driver->get(cpu);
2200                 if (WARN_ON(!new_policy.cur)) {
2201                         ret = -EIO;
2202                         goto no_policy;
2203                 }
2204
2205                 if (!policy->cur) {
2206                         pr_debug("Driver did not initialize current freq\n");
2207                         policy->cur = new_policy.cur;
2208                 } else {
2209                         if (policy->cur != new_policy.cur && has_target())
2210                                 cpufreq_out_of_sync(cpu, policy->cur,
2211                                                                 new_policy.cur);
2212                 }
2213         }
2214
2215         ret = cpufreq_set_policy(policy, &new_policy);
2216
2217         up_write(&policy->rwsem);
2218
2219         cpufreq_cpu_put(policy);
2220 no_policy:
2221         return ret;
2222 }
2223 EXPORT_SYMBOL(cpufreq_update_policy);
2224
2225 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2226                                         unsigned long action, void *hcpu)
2227 {
2228         unsigned int cpu = (unsigned long)hcpu;
2229         struct device *dev;
2230
2231         dev = get_cpu_device(cpu);
2232         if (dev) {
2233                 switch (action & ~CPU_TASKS_FROZEN) {
2234                 case CPU_ONLINE:
2235                         __cpufreq_add_dev(dev, NULL);
2236                         break;
2237
2238                 case CPU_DOWN_PREPARE:
2239                         __cpufreq_remove_dev_prepare(dev, NULL);
2240                         break;
2241
2242                 case CPU_POST_DEAD:
2243                         __cpufreq_remove_dev_finish(dev, NULL);
2244                         break;
2245
2246                 case CPU_DOWN_FAILED:
2247                         __cpufreq_add_dev(dev, NULL);
2248                         break;
2249                 }
2250         }
2251         return NOTIFY_OK;
2252 }
2253
2254 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2255         .notifier_call = cpufreq_cpu_callback,
2256 };
2257
2258 /*********************************************************************
2259  *               BOOST                                               *
2260  *********************************************************************/
2261 static int cpufreq_boost_set_sw(int state)
2262 {
2263         struct cpufreq_frequency_table *freq_table;
2264         struct cpufreq_policy *policy;
2265         int ret = -EINVAL;
2266
2267         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2268                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2269                 if (freq_table) {
2270                         ret = cpufreq_frequency_table_cpuinfo(policy,
2271                                                         freq_table);
2272                         if (ret) {
2273                                 pr_err("%s: Policy frequency update failed\n",
2274                                        __func__);
2275                                 break;
2276                         }
2277                         policy->user_policy.max = policy->max;
2278                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2279                 }
2280         }
2281
2282         return ret;
2283 }
2284
2285 int cpufreq_boost_trigger_state(int state)
2286 {
2287         unsigned long flags;
2288         int ret = 0;
2289
2290         if (cpufreq_driver->boost_enabled == state)
2291                 return 0;
2292
2293         write_lock_irqsave(&cpufreq_driver_lock, flags);
2294         cpufreq_driver->boost_enabled = state;
2295         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2296
2297         ret = cpufreq_driver->set_boost(state);
2298         if (ret) {
2299                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2300                 cpufreq_driver->boost_enabled = !state;
2301                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2302
2303                 pr_err("%s: Cannot %s BOOST\n",
2304                        __func__, state ? "enable" : "disable");
2305         }
2306
2307         return ret;
2308 }
2309
2310 int cpufreq_boost_supported(void)
2311 {
2312         if (likely(cpufreq_driver))
2313                 return cpufreq_driver->boost_supported;
2314
2315         return 0;
2316 }
2317 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2318
2319 int cpufreq_boost_enabled(void)
2320 {
2321         return cpufreq_driver->boost_enabled;
2322 }
2323 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2324
2325 /*********************************************************************
2326  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2327  *********************************************************************/
2328
2329 /**
2330  * cpufreq_register_driver - register a CPU Frequency driver
2331  * @driver_data: A struct cpufreq_driver containing the values#
2332  * submitted by the CPU Frequency driver.
2333  *
2334  * Registers a CPU Frequency driver to this core code. This code
2335  * returns zero on success, -EBUSY when another driver got here first
2336  * (and isn't unregistered in the meantime).
2337  *
2338  */
2339 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2340 {
2341         unsigned long flags;
2342         int ret;
2343
2344         if (cpufreq_disabled())
2345                 return -ENODEV;
2346
2347         if (!driver_data || !driver_data->verify || !driver_data->init ||
2348             !(driver_data->setpolicy || driver_data->target_index ||
2349                     driver_data->target) ||
2350              (driver_data->setpolicy && (driver_data->target_index ||
2351                     driver_data->target)))
2352                 return -EINVAL;
2353
2354         pr_debug("trying to register driver %s\n", driver_data->name);
2355
2356         if (driver_data->setpolicy)
2357                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2358
2359         write_lock_irqsave(&cpufreq_driver_lock, flags);
2360         if (cpufreq_driver) {
2361                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2362                 return -EEXIST;
2363         }
2364         cpufreq_driver = driver_data;
2365         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2366
2367         if (cpufreq_boost_supported()) {
2368                 /*
2369                  * Check if driver provides function to enable boost -
2370                  * if not, use cpufreq_boost_set_sw as default
2371                  */
2372                 if (!cpufreq_driver->set_boost)
2373                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2374
2375                 ret = cpufreq_sysfs_create_file(&boost.attr);
2376                 if (ret) {
2377                         pr_err("%s: cannot register global BOOST sysfs file\n",
2378                                __func__);
2379                         goto err_null_driver;
2380                 }
2381         }
2382
2383         ret = subsys_interface_register(&cpufreq_interface);
2384         if (ret)
2385                 goto err_boost_unreg;
2386
2387         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2388                 int i;
2389                 ret = -ENODEV;
2390
2391                 /* check for at least one working CPU */
2392                 for (i = 0; i < nr_cpu_ids; i++)
2393                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2394                                 ret = 0;
2395                                 break;
2396                         }
2397
2398                 /* if all ->init() calls failed, unregister */
2399                 if (ret) {
2400                         pr_debug("no CPU initialized for driver %s\n",
2401                                  driver_data->name);
2402                         goto err_if_unreg;
2403                 }
2404         }
2405
2406         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2407         pr_debug("driver %s up and running\n", driver_data->name);
2408
2409         return 0;
2410 err_if_unreg:
2411         subsys_interface_unregister(&cpufreq_interface);
2412 err_boost_unreg:
2413         if (cpufreq_boost_supported())
2414                 cpufreq_sysfs_remove_file(&boost.attr);
2415 err_null_driver:
2416         write_lock_irqsave(&cpufreq_driver_lock, flags);
2417         cpufreq_driver = NULL;
2418         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2419         return ret;
2420 }
2421 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2422
2423 /**
2424  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2425  *
2426  * Unregister the current CPUFreq driver. Only call this if you have
2427  * the right to do so, i.e. if you have succeeded in initialising before!
2428  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2429  * currently not initialised.
2430  */
2431 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2432 {
2433         unsigned long flags;
2434
2435         if (!cpufreq_driver || (driver != cpufreq_driver))
2436                 return -EINVAL;
2437
2438         pr_debug("unregistering driver %s\n", driver->name);
2439
2440         subsys_interface_unregister(&cpufreq_interface);
2441         if (cpufreq_boost_supported())
2442                 cpufreq_sysfs_remove_file(&boost.attr);
2443
2444         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2445
2446         down_write(&cpufreq_rwsem);
2447         write_lock_irqsave(&cpufreq_driver_lock, flags);
2448
2449         cpufreq_driver = NULL;
2450
2451         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2452         up_write(&cpufreq_rwsem);
2453
2454         return 0;
2455 }
2456 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2457
2458 static int __init cpufreq_core_init(void)
2459 {
2460         if (cpufreq_disabled())
2461                 return -ENODEV;
2462
2463         cpufreq_global_kobject = kobject_create();
2464         BUG_ON(!cpufreq_global_kobject);
2465
2466         return 0;
2467 }
2468 core_initcall(cpufreq_core_init);