drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/suspend.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 /* This one keeps track of the previously set governor of a removed CPU */
  46 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  47
  48 /* Flag to suspend/resume CPUFreq governors */
  49 static bool cpufreq_suspended;
  50
  51 static inline bool has_target(void)
  52 {
  53         return cpufreq_driver->target_index || cpufreq_driver->target;
  54 }
  55
  56 /*
  57  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  58  * sections
  59  */
  60 static DECLARE_RWSEM(cpufreq_rwsem);
  61
  62 /* internal prototypes */
  63 static int __cpufreq_governor(struct cpufreq_policy *policy,
  64                 unsigned int event);
  65 static unsigned int __cpufreq_get(unsigned int cpu);
  66 static void handle_update(struct work_struct *work);
  67
  68 /**
  69  * Two notifier lists: the "policy" list is involved in the
  70  * validation process for a new CPU frequency policy; the
  71  * "transition" list for kernel code that needs to handle
  72  * changes to devices when the CPU clock speed changes.
  73  * The mutex locks both lists.
  74  */
  75 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  76 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  77
  78 static bool init_cpufreq_transition_notifier_list_called;
  79 static int __init init_cpufreq_transition_notifier_list(void)
  80 {
  81         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  82         init_cpufreq_transition_notifier_list_called = true;
  83         return 0;
  84 }
  85 pure_initcall(init_cpufreq_transition_notifier_list);
  86
  87 static int off __read_mostly;
  88 static int cpufreq_disabled(void)
  89 {
  90         return off;
  91 }
  92 void disable_cpufreq(void)
  93 {
  94         off = 1;
  95 }
  96 static LIST_HEAD(cpufreq_governor_list);
  97 static DEFINE_MUTEX(cpufreq_governor_mutex);
  98
  99 bool have_governor_per_policy(void)
 100 {
 101         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 102 }
 103 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 104
 105 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 106 {
 107         if (have_governor_per_policy())
 108                 return &policy->kobj;
 109         else
 110                 return cpufreq_global_kobject;
 111 }
 112 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 113
 114 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 115 {
 116         u64 idle_time;
 117         u64 cur_wall_time;
 118         u64 busy_time;
 119
 120         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 121
 122         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 127         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 128
 129         idle_time = cur_wall_time - busy_time;
 130         if (wall)
 131                 *wall = cputime_to_usecs(cur_wall_time);
 132
 133         return cputime_to_usecs(idle_time);
 134 }
 135
 136 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 137 {
 138         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 139
 140         if (idle_time == -1ULL)
 141                 return get_cpu_idle_time_jiffy(cpu, wall);
 142         else if (!io_busy)
 143                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 144
 145         return idle_time;
 146 }
 147 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 148
 149 /*
 150  * This is a generic cpufreq init() routine which can be used by cpufreq
 151  * drivers of SMP systems. It will do following:
 152  * - validate & show freq table passed
 153  * - set policies transition latency
 154  * - policy->cpus with all possible CPUs
 155  */
 156 int cpufreq_generic_init(struct cpufreq_policy *policy,
 157                 struct cpufreq_frequency_table *table,
 158                 unsigned int transition_latency)
 159 {
 160         int ret;
 161
 162         ret = cpufreq_table_validate_and_show(policy, table);
 163         if (ret) {
 164                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 165                 return ret;
 166         }
 167
 168         policy->cpuinfo.transition_latency = transition_latency;
 169
 170         /*
 171          * The driver only supports the SMP configuartion where all processors
 172          * share the clock and voltage and clock.
 173          */
 174         cpumask_setall(policy->cpus);
 175
 176         return 0;
 177 }
 178 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 179
 180 unsigned int cpufreq_generic_get(unsigned int cpu)
 181 {
 182         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
 183
 184         if (!policy || IS_ERR(policy->clk)) {
 185                 pr_err("%s: No %s associated to cpu: %d\n",
 186                        __func__, policy ? "clk" : "policy", cpu);
 187                 return 0;
 188         }
 189
 190         return clk_get_rate(policy->clk) / 1000;
 191 }
 192 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
 193
 194 /* Only for cpufreq core internal use */
 195 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
 196 {
 197         return per_cpu(cpufreq_cpu_data, cpu);
 198 }
 199
 200 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 201 {
 202         struct cpufreq_policy *policy = NULL;
 203         unsigned long flags;
 204
 205         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 206                 return NULL;
 207
 208         if (!down_read_trylock(&cpufreq_rwsem))
 209                 return NULL;
 210
 211         /* get the cpufreq driver */
 212         read_lock_irqsave(&cpufreq_driver_lock, flags);
 213
 214         if (cpufreq_driver) {
 215                 /* get the CPU */
 216                 policy = per_cpu(cpufreq_cpu_data, cpu);
 217                 if (policy)
 218                         kobject_get(&policy->kobj);
 219         }
 220
 221         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 222
 223         if (!policy)
 224                 up_read(&cpufreq_rwsem);
 225
 226         return policy;
 227 }
 228 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 229
 230 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 231 {
 232         if (cpufreq_disabled())
 233                 return;
 234
 235         kobject_put(&policy->kobj);
 236         up_read(&cpufreq_rwsem);
 237 }
 238 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 239
 240 /*********************************************************************
 241  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 242  *********************************************************************/
 243
 244 /**
 245  * adjust_jiffies - adjust the system "loops_per_jiffy"
 246  *
 247  * This function alters the system "loops_per_jiffy" for the clock
 248  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 249  * systems as each CPU might be scaled differently. So, use the arch
 250  * per-CPU loops_per_jiffy value wherever possible.
 251  */
 252 #ifndef CONFIG_SMP
 253 static unsigned long l_p_j_ref;
 254 static unsigned int l_p_j_ref_freq;
 255
 256 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 257 {
 258         if (ci->flags & CPUFREQ_CONST_LOOPS)
 259                 return;
 260
 261         if (!l_p_j_ref_freq) {
 262                 l_p_j_ref = loops_per_jiffy;
 263                 l_p_j_ref_freq = ci->old;
 264                 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
 265                          l_p_j_ref, l_p_j_ref_freq);
 266         }
 267         if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
 268                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 269                                                                 ci->new);
 270                 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
 271                          loops_per_jiffy, ci->new);
 272         }
 273 }
 274 #else
 275 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 276 {
 277         return;
 278 }
 279 #endif
 280
 281 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 282                 struct cpufreq_freqs *freqs, unsigned int state)
 283 {
 284         BUG_ON(irqs_disabled());
 285
 286         if (cpufreq_disabled())
 287                 return;
 288
 289         freqs->flags = cpufreq_driver->flags;
 290         pr_debug("notification %u of frequency transition to %u kHz\n",
 291                  state, freqs->new);
 292
 293         switch (state) {
 294
 295         case CPUFREQ_PRECHANGE:
 296                 /* detect if the driver reported a value as "old frequency"
 297                  * which is not equal to what the cpufreq core thinks is
 298                  * "old frequency".
 299                  */
 300                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 301                         if ((policy) && (policy->cpu == freqs->cpu) &&
 302                             (policy->cur) && (policy->cur != freqs->old)) {
 303                                 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
 304                                          freqs->old, policy->cur);
 305                                 freqs->old = policy->cur;
 306                         }
 307                 }
 308                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 309                                 CPUFREQ_PRECHANGE, freqs);
 310                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 311                 break;
 312
 313         case CPUFREQ_POSTCHANGE:
 314                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 315                 pr_debug("FREQ: %lu - CPU: %lu\n",
 316                          (unsigned long)freqs->new, (unsigned long)freqs->cpu);
 317                 trace_cpu_frequency(freqs->new, freqs->cpu);
 318                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 319                                 CPUFREQ_POSTCHANGE, freqs);
 320                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 321                         policy->cur = freqs->new;
 322                 break;
 323         }
 324 }
 325
 326 /**
 327  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 328  * on frequency transition.
 329  *
 330  * This function calls the transition notifiers and the "adjust_jiffies"
 331  * function. It is called twice on all CPU frequency changes that have
 332  * external effects.
 333  */
 334 static void cpufreq_notify_transition(struct cpufreq_policy *policy,
 335                 struct cpufreq_freqs *freqs, unsigned int state)
 336 {
 337         for_each_cpu(freqs->cpu, policy->cpus)
 338                 __cpufreq_notify_transition(policy, freqs, state);
 339 }
 340
 341 /* Do post notifications when there are chances that transition has failed */
 342 static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
 343                 struct cpufreq_freqs *freqs, int transition_failed)
 344 {
 345         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 346         if (!transition_failed)
 347                 return;
 348
 349         swap(freqs->old, freqs->new);
 350         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 351         cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
 352 }
 353
 354 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
 355                 struct cpufreq_freqs *freqs)
 356 {
 357
 358         /*
 359          * Catch double invocations of _begin() which lead to self-deadlock.
 360          * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
 361          * doesn't invoke _begin() on their behalf, and hence the chances of
 362          * double invocations are very low. Moreover, there are scenarios
 363          * where these checks can emit false-positive warnings in these
 364          * drivers; so we avoid that by skipping them altogether.
 365          */
 366         WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
 367                                 && current == policy->transition_task);
 368
 369 wait:
 370         wait_event(policy->transition_wait, !policy->transition_ongoing);
 371
 372         spin_lock(&policy->transition_lock);
 373
 374         if (unlikely(policy->transition_ongoing)) {
 375                 spin_unlock(&policy->transition_lock);
 376                 goto wait;
 377         }
 378
 379         policy->transition_ongoing = true;
 380         policy->transition_task = current;
 381
 382         spin_unlock(&policy->transition_lock);
 383
 384         cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
 385 }
 386 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
 387
 388 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
 389                 struct cpufreq_freqs *freqs, int transition_failed)
 390 {
 391         if (unlikely(WARN_ON(!policy->transition_ongoing)))
 392                 return;
 393
 394         cpufreq_notify_post_transition(policy, freqs, transition_failed);
 395
 396         policy->transition_ongoing = false;
 397         policy->transition_task = NULL;
 398
 399         wake_up(&policy->transition_wait);
 400 }
 401 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
 402
 403
 404 /*********************************************************************
 405  *                          SYSFS INTERFACE                          *
 406  *********************************************************************/
 407 static ssize_t show_boost(struct kobject *kobj,
 408                                  struct attribute *attr, char *buf)
 409 {
 410         return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
 411 }
 412
 413 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 414                                   const char *buf, size_t count)
 415 {
 416         int ret, enable;
 417
 418         ret = sscanf(buf, "%d", &enable);
 419         if (ret != 1 || enable < 0 || enable > 1)
 420                 return -EINVAL;
 421
 422         if (cpufreq_boost_trigger_state(enable)) {
 423                 pr_err("%s: Cannot %s BOOST!\n",
 424                        __func__, enable ? "enable" : "disable");
 425                 return -EINVAL;
 426         }
 427
 428         pr_debug("%s: cpufreq BOOST %s\n",
 429                  __func__, enable ? "enabled" : "disabled");
 430
 431         return count;
 432 }
 433 define_one_global_rw(boost);
 434
 435 static struct cpufreq_governor *__find_governor(const char *str_governor)
 436 {
 437         struct cpufreq_governor *t;
 438
 439         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 440                 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 441                         return t;
 442
 443         return NULL;
 444 }
 445
 446 /**
 447  * cpufreq_parse_governor - parse a governor string
 448  */
 449 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 450                                 struct cpufreq_governor **governor)
 451 {
 452         int err = -EINVAL;
 453
 454         if (!cpufreq_driver)
 455                 goto out;
 456
 457         if (cpufreq_driver->setpolicy) {
 458                 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 459                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 460                         err = 0;
 461                 } else if (!strncasecmp(str_governor, "powersave",
 462                                                 CPUFREQ_NAME_LEN)) {
 463                         *policy = CPUFREQ_POLICY_POWERSAVE;
 464                         err = 0;
 465                 }
 466         } else if (has_target()) {
 467                 struct cpufreq_governor *t;
 468
 469                 mutex_lock(&cpufreq_governor_mutex);
 470
 471                 t = __find_governor(str_governor);
 472
 473                 if (t == NULL) {
 474                         int ret;
 475
 476                         mutex_unlock(&cpufreq_governor_mutex);
 477                         ret = request_module("cpufreq_%s", str_governor);
 478                         mutex_lock(&cpufreq_governor_mutex);
 479
 480                         if (ret == 0)
 481                                 t = __find_governor(str_governor);
 482                 }
 483
 484                 if (t != NULL) {
 485                         *governor = t;
 486                         err = 0;
 487                 }
 488
 489                 mutex_unlock(&cpufreq_governor_mutex);
 490         }
 491 out:
 492         return err;
 493 }
 494
 495 /**
 496  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 497  * print out cpufreq information
 498  *
 499  * Write out information from cpufreq_driver->policy[cpu]; object must be
 500  * "unsigned int".
 501  */
 502
 503 #define show_one(file_name, object)                     \
 504 static ssize_t show_##file_name                         \
 505 (struct cpufreq_policy *policy, char *buf)              \
 506 {                                                       \
 507         return sprintf(buf, "%u\n", policy->object);    \
 508 }
 509
 510 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 511 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 512 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 513 show_one(scaling_min_freq, min);
 514 show_one(scaling_max_freq, max);
 515
 516 static ssize_t show_scaling_cur_freq(
 517         struct cpufreq_policy *policy, char *buf)
 518 {
 519         ssize_t ret;
 520
 521         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
 522                 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
 523         else
 524                 ret = sprintf(buf, "%u\n", policy->cur);
 525         return ret;
 526 }
 527
 528 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 529                                 struct cpufreq_policy *new_policy);
 530
 531 /**
 532  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 533  */
 534 #define store_one(file_name, object)                    \
 535 static ssize_t store_##file_name                                        \
 536 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 537 {                                                                       \
 538         int ret, temp;                                                  \
 539         struct cpufreq_policy new_policy;                               \
 540                                                                         \
 541         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 542         if (ret)                                                        \
 543                 return -EINVAL;                                         \
 544                                                                         \
 545         ret = sscanf(buf, "%u", &new_policy.object);                    \
 546         if (ret != 1)                                                   \
 547                 return -EINVAL;                                         \
 548                                                                         \
 549         temp = new_policy.object;                                       \
 550         ret = cpufreq_set_policy(policy, &new_policy);          \
 551         if (!ret)                                                       \
 552                 policy->user_policy.object = temp;                      \
 553                                                                         \
 554         return ret ? ret : count;                                       \
 555 }
 556
 557 store_one(scaling_min_freq, min);
 558 store_one(scaling_max_freq, max);
 559
 560 /**
 561  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 562  */
 563 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 564                                         char *buf)
 565 {
 566         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 567         if (!cur_freq)
 568                 return sprintf(buf, "<unknown>");
 569         return sprintf(buf, "%u\n", cur_freq);
 570 }
 571
 572 /**
 573  * show_scaling_governor - show the current policy for the specified CPU
 574  */
 575 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 576 {
 577         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 578                 return sprintf(buf, "powersave\n");
 579         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 580                 return sprintf(buf, "performance\n");
 581         else if (policy->governor)
 582                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 583                                 policy->governor->name);
 584         return -EINVAL;
 585 }
 586
 587 /**
 588  * store_scaling_governor - store policy for the specified CPU
 589  */
 590 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 591                                         const char *buf, size_t count)
 592 {
 593         int ret;
 594         char    str_governor[16];
 595         struct cpufreq_policy new_policy;
 596
 597         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 598         if (ret)
 599                 return ret;
 600
 601         ret = sscanf(buf, "%15s", str_governor);
 602         if (ret != 1)
 603                 return -EINVAL;
 604
 605         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 606                                                 &new_policy.governor))
 607                 return -EINVAL;
 608
 609         ret = cpufreq_set_policy(policy, &new_policy);
 610
 611         policy->user_policy.policy = policy->policy;
 612         policy->user_policy.governor = policy->governor;
 613
 614         if (ret)
 615                 return ret;
 616         else
 617                 return count;
 618 }
 619
 620 /**
 621  * show_scaling_driver - show the cpufreq driver currently loaded
 622  */
 623 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 624 {
 625         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 626 }
 627
 628 /**
 629  * show_scaling_available_governors - show the available CPUfreq governors
 630  */
 631 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 632                                                 char *buf)
 633 {
 634         ssize_t i = 0;
 635         struct cpufreq_governor *t;
 636
 637         if (!has_target()) {
 638                 i += sprintf(buf, "performance powersave");
 639                 goto out;
 640         }
 641
 642         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 643                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 644                     - (CPUFREQ_NAME_LEN + 2)))
 645                         goto out;
 646                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 647         }
 648 out:
 649         i += sprintf(&buf[i], "\n");
 650         return i;
 651 }
 652
 653 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 654 {
 655         ssize_t i = 0;
 656         unsigned int cpu;
 657
 658         for_each_cpu(cpu, mask) {
 659                 if (i)
 660                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 661                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 662                 if (i >= (PAGE_SIZE - 5))
 663                         break;
 664         }
 665         i += sprintf(&buf[i], "\n");
 666         return i;
 667 }
 668 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 669
 670 /**
 671  * show_related_cpus - show the CPUs affected by each transition even if
 672  * hw coordination is in use
 673  */
 674 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 675 {
 676         return cpufreq_show_cpus(policy->related_cpus, buf);
 677 }
 678
 679 /**
 680  * show_affected_cpus - show the CPUs affected by each transition
 681  */
 682 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 683 {
 684         return cpufreq_show_cpus(policy->cpus, buf);
 685 }
 686
 687 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 688                                         const char *buf, size_t count)
 689 {
 690         unsigned int freq = 0;
 691         unsigned int ret;
 692
 693         if (!policy->governor || !policy->governor->store_setspeed)
 694                 return -EINVAL;
 695
 696         ret = sscanf(buf, "%u", &freq);
 697         if (ret != 1)
 698                 return -EINVAL;
 699
 700         policy->governor->store_setspeed(policy, freq);
 701
 702         return count;
 703 }
 704
 705 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 706 {
 707         if (!policy->governor || !policy->governor->show_setspeed)
 708                 return sprintf(buf, "<unsupported>\n");
 709
 710         return policy->governor->show_setspeed(policy, buf);
 711 }
 712
 713 /**
 714  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 715  */
 716 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 717 {
 718         unsigned int limit;
 719         int ret;
 720         if (cpufreq_driver->bios_limit) {
 721                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 722                 if (!ret)
 723                         return sprintf(buf, "%u\n", limit);
 724         }
 725         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 726 }
 727
 728 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 729 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 730 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 731 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 732 cpufreq_freq_attr_ro(scaling_available_governors);
 733 cpufreq_freq_attr_ro(scaling_driver);
 734 cpufreq_freq_attr_ro(scaling_cur_freq);
 735 cpufreq_freq_attr_ro(bios_limit);
 736 cpufreq_freq_attr_ro(related_cpus);
 737 cpufreq_freq_attr_ro(affected_cpus);
 738 cpufreq_freq_attr_rw(scaling_min_freq);
 739 cpufreq_freq_attr_rw(scaling_max_freq);
 740 cpufreq_freq_attr_rw(scaling_governor);
 741 cpufreq_freq_attr_rw(scaling_setspeed);
 742
 743 static struct attribute *default_attrs[] = {
 744         &cpuinfo_min_freq.attr,
 745         &cpuinfo_max_freq.attr,
 746         &cpuinfo_transition_latency.attr,
 747         &scaling_min_freq.attr,
 748         &scaling_max_freq.attr,
 749         &affected_cpus.attr,
 750         &related_cpus.attr,
 751         &scaling_governor.attr,
 752         &scaling_driver.attr,
 753         &scaling_available_governors.attr,
 754         &scaling_setspeed.attr,
 755         NULL
 756 };
 757
 758 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 759 #define to_attr(a) container_of(a, struct freq_attr, attr)
 760
 761 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 762 {
 763         struct cpufreq_policy *policy = to_policy(kobj);
 764         struct freq_attr *fattr = to_attr(attr);
 765         ssize_t ret;
 766
 767         if (!down_read_trylock(&cpufreq_rwsem))
 768                 return -EINVAL;
 769
 770         down_read(&policy->rwsem);
 771
 772         if (fattr->show)
 773                 ret = fattr->show(policy, buf);
 774         else
 775                 ret = -EIO;
 776
 777         up_read(&policy->rwsem);
 778         up_read(&cpufreq_rwsem);
 779
 780         return ret;
 781 }
 782
 783 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 784                      const char *buf, size_t count)
 785 {
 786         struct cpufreq_policy *policy = to_policy(kobj);
 787         struct freq_attr *fattr = to_attr(attr);
 788         ssize_t ret = -EINVAL;
 789
 790         get_online_cpus();
 791
 792         if (!cpu_online(policy->cpu))
 793                 goto unlock;
 794
 795         if (!down_read_trylock(&cpufreq_rwsem))
 796                 goto unlock;
 797
 798         down_write(&policy->rwsem);
 799
 800         if (fattr->store)
 801                 ret = fattr->store(policy, buf, count);
 802         else
 803                 ret = -EIO;
 804
 805         up_write(&policy->rwsem);
 806
 807         up_read(&cpufreq_rwsem);
 808 unlock:
 809         put_online_cpus();
 810
 811         return ret;
 812 }
 813
 814 static void cpufreq_sysfs_release(struct kobject *kobj)
 815 {
 816         struct cpufreq_policy *policy = to_policy(kobj);
 817         pr_debug("last reference is dropped\n");
 818         complete(&policy->kobj_unregister);
 819 }
 820
 821 static const struct sysfs_ops sysfs_ops = {
 822         .show   = show,
 823         .store  = store,
 824 };
 825
 826 static struct kobj_type ktype_cpufreq = {
 827         .sysfs_ops      = &sysfs_ops,
 828         .default_attrs  = default_attrs,
 829         .release        = cpufreq_sysfs_release,
 830 };
 831
 832 struct kobject *cpufreq_global_kobject;
 833 EXPORT_SYMBOL(cpufreq_global_kobject);
 834
 835 static int cpufreq_global_kobject_usage;
 836
 837 int cpufreq_get_global_kobject(void)
 838 {
 839         if (!cpufreq_global_kobject_usage++)
 840                 return kobject_add(cpufreq_global_kobject,
 841                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 842
 843         return 0;
 844 }
 845 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 846
 847 void cpufreq_put_global_kobject(void)
 848 {
 849         if (!--cpufreq_global_kobject_usage)
 850                 kobject_del(cpufreq_global_kobject);
 851 }
 852 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 853
 854 int cpufreq_sysfs_create_file(const struct attribute *attr)
 855 {
 856         int ret = cpufreq_get_global_kobject();
 857
 858         if (!ret) {
 859                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 860                 if (ret)
 861                         cpufreq_put_global_kobject();
 862         }
 863
 864         return ret;
 865 }
 866 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 867
 868 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 869 {
 870         sysfs_remove_file(cpufreq_global_kobject, attr);
 871         cpufreq_put_global_kobject();
 872 }
 873 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 874
 875 /* symlink affected CPUs */
 876 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 877 {
 878         unsigned int j;
 879         int ret = 0;
 880
 881         for_each_cpu(j, policy->cpus) {
 882                 struct device *cpu_dev;
 883
 884                 if (j == policy->cpu)
 885                         continue;
 886
 887                 pr_debug("Adding link for CPU: %u\n", j);
 888                 cpu_dev = get_cpu_device(j);
 889                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 890                                         "cpufreq");
 891                 if (ret)
 892                         break;
 893         }
 894         return ret;
 895 }
 896
 897 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 898                                      struct device *dev)
 899 {
 900         struct freq_attr **drv_attr;
 901         int ret = 0;
 902
 903         /* set up files for this cpu device */
 904         drv_attr = cpufreq_driver->attr;
 905         while ((drv_attr) && (*drv_attr)) {
 906                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 907                 if (ret)
 908                         return ret;
 909                 drv_attr++;
 910         }
 911         if (cpufreq_driver->get) {
 912                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 913                 if (ret)
 914                         return ret;
 915         }
 916
 917         ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 918         if (ret)
 919                 return ret;
 920
 921         if (cpufreq_driver->bios_limit) {
 922                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 923                 if (ret)
 924                         return ret;
 925         }
 926
 927         return cpufreq_add_dev_symlink(policy);
 928 }
 929
 930 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 931 {
 932         struct cpufreq_governor *gov = NULL;
 933         struct cpufreq_policy new_policy;
 934         int ret = 0;
 935
 936         memcpy(&new_policy, policy, sizeof(*policy));
 937
 938         /* Update governor of new_policy to the governor used before hotplug */
 939         gov = __find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
 940         if (gov)
 941                 pr_debug("Restoring governor %s for cpu %d\n",
 942                                 policy->governor->name, policy->cpu);
 943         else
 944                 gov = CPUFREQ_DEFAULT_GOVERNOR;
 945
 946         new_policy.governor = gov;
 947
 948         /* Use the default policy if its valid. */
 949         if (cpufreq_driver->setpolicy)
 950                 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
 951
 952         /* set default policy */
 953         ret = cpufreq_set_policy(policy, &new_policy);
 954         if (ret) {
 955                 pr_debug("setting policy failed\n");
 956                 if (cpufreq_driver->exit)
 957                         cpufreq_driver->exit(policy);
 958         }
 959 }
 960
 961 #ifdef CONFIG_HOTPLUG_CPU
 962 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 963                                   unsigned int cpu, struct device *dev)
 964 {
 965         int ret = 0;
 966         unsigned long flags;
 967
 968         if (has_target()) {
 969                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 970                 if (ret) {
 971                         pr_err("%s: Failed to stop governor\n", __func__);
 972                         return ret;
 973                 }
 974         }
 975
 976         down_write(&policy->rwsem);
 977
 978         write_lock_irqsave(&cpufreq_driver_lock, flags);
 979
 980         cpumask_set_cpu(cpu, policy->cpus);
 981         per_cpu(cpufreq_cpu_data, cpu) = policy;
 982         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 983
 984         up_write(&policy->rwsem);
 985
 986         if (has_target()) {
 987                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
 988                 if (!ret)
 989                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 990
 991                 if (ret) {
 992                         pr_err("%s: Failed to start governor\n", __func__);
 993                         return ret;
 994                 }
 995         }
 996
 997         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 998 }
 999 #endif
1000
1001 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
1002 {
1003         struct cpufreq_policy *policy;
1004         unsigned long flags;
1005
1006         read_lock_irqsave(&cpufreq_driver_lock, flags);
1007
1008         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
1009
1010         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1011
1012         if (policy)
1013                 policy->governor = NULL;
1014
1015         return policy;
1016 }
1017
1018 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1019 {
1020         struct cpufreq_policy *policy;
1021
1022         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1023         if (!policy)
1024                 return NULL;
1025
1026         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1027                 goto err_free_policy;
1028
1029         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1030                 goto err_free_cpumask;
1031
1032         INIT_LIST_HEAD(&policy->policy_list);
1033         init_rwsem(&policy->rwsem);
1034         spin_lock_init(&policy->transition_lock);
1035         init_waitqueue_head(&policy->transition_wait);
1036
1037         return policy;
1038
1039 err_free_cpumask:
1040         free_cpumask_var(policy->cpus);
1041 err_free_policy:
1042         kfree(policy);
1043
1044         return NULL;
1045 }
1046
1047 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1048 {
1049         struct kobject *kobj;
1050         struct completion *cmp;
1051
1052         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1053                         CPUFREQ_REMOVE_POLICY, policy);
1054
1055         down_read(&policy->rwsem);
1056         kobj = &policy->kobj;
1057         cmp = &policy->kobj_unregister;
1058         up_read(&policy->rwsem);
1059         kobject_put(kobj);
1060
1061         /*
1062          * We need to make sure that the underlying kobj is
1063          * actually not referenced anymore by anybody before we
1064          * proceed with unloading.
1065          */
1066         pr_debug("waiting for dropping of refcount\n");
1067         wait_for_completion(cmp);
1068         pr_debug("wait complete\n");
1069 }
1070
1071 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1072 {
1073         free_cpumask_var(policy->related_cpus);
1074         free_cpumask_var(policy->cpus);
1075         kfree(policy);
1076 }
1077
1078 static int update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu,
1079                              struct device *cpu_dev)
1080 {
1081         int ret;
1082
1083         if (WARN_ON(cpu == policy->cpu))
1084                 return 0;
1085
1086         /* Move kobject to the new policy->cpu */
1087         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1088         if (ret) {
1089                 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1090                 return ret;
1091         }
1092
1093         down_write(&policy->rwsem);
1094
1095         policy->last_cpu = policy->cpu;
1096         policy->cpu = cpu;
1097
1098         up_write(&policy->rwsem);
1099
1100         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1101                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1102
1103         return 0;
1104 }
1105
1106 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1107 {
1108         unsigned int j, cpu = dev->id;
1109         int ret = -ENOMEM;
1110         struct cpufreq_policy *policy;
1111         unsigned long flags;
1112         bool recover_policy = cpufreq_suspended;
1113 #ifdef CONFIG_HOTPLUG_CPU
1114         struct cpufreq_policy *tpolicy;
1115 #endif
1116
1117         if (cpu_is_offline(cpu))
1118                 return 0;
1119
1120         pr_debug("adding CPU %u\n", cpu);
1121
1122 #ifdef CONFIG_SMP
1123         /* check whether a different CPU already registered this
1124          * CPU because it is in the same boat. */
1125         policy = cpufreq_cpu_get(cpu);
1126         if (unlikely(policy)) {
1127                 cpufreq_cpu_put(policy);
1128                 return 0;
1129         }
1130 #endif
1131
1132         if (!down_read_trylock(&cpufreq_rwsem))
1133                 return 0;
1134
1135 #ifdef CONFIG_HOTPLUG_CPU
1136         /* Check if this cpu was hot-unplugged earlier and has siblings */
1137         read_lock_irqsave(&cpufreq_driver_lock, flags);
1138         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1139                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1140                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1141                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1142                         up_read(&cpufreq_rwsem);
1143                         return ret;
1144                 }
1145         }
1146         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1147 #endif
1148
1149         /*
1150          * Restore the saved policy when doing light-weight init and fall back
1151          * to the full init if that fails.
1152          */
1153         policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1154         if (!policy) {
1155                 recover_policy = false;
1156                 policy = cpufreq_policy_alloc();
1157                 if (!policy)
1158                         goto nomem_out;
1159         }
1160
1161         /*
1162          * In the resume path, since we restore a saved policy, the assignment
1163          * to policy->cpu is like an update of the existing policy, rather than
1164          * the creation of a brand new one. So we need to perform this update
1165          * by invoking update_policy_cpu().
1166          */
1167         if (recover_policy && cpu != policy->cpu)
1168                 WARN_ON(update_policy_cpu(policy, cpu, dev));
1169         else
1170                 policy->cpu = cpu;
1171
1172         cpumask_copy(policy->cpus, cpumask_of(cpu));
1173
1174         init_completion(&policy->kobj_unregister);
1175         INIT_WORK(&policy->update, handle_update);
1176
1177         /* call driver. From then on the cpufreq must be able
1178          * to accept all calls to ->verify and ->setpolicy for this CPU
1179          */
1180         ret = cpufreq_driver->init(policy);
1181         if (ret) {
1182                 pr_debug("initialization failed\n");
1183                 goto err_set_policy_cpu;
1184         }
1185
1186         down_write(&policy->rwsem);
1187
1188         /* related cpus should atleast have policy->cpus */
1189         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1190
1191         /*
1192          * affected cpus must always be the one, which are online. We aren't
1193          * managing offline cpus here.
1194          */
1195         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1196
1197         if (!recover_policy) {
1198                 policy->user_policy.min = policy->min;
1199                 policy->user_policy.max = policy->max;
1200
1201                 /* prepare interface data */
1202                 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1203                                            &dev->kobj, "cpufreq");
1204                 if (ret) {
1205                         pr_err("%s: failed to init policy->kobj: %d\n",
1206                                __func__, ret);
1207                         goto err_init_policy_kobj;
1208                 }
1209         }
1210
1211         write_lock_irqsave(&cpufreq_driver_lock, flags);
1212         for_each_cpu(j, policy->cpus)
1213                 per_cpu(cpufreq_cpu_data, j) = policy;
1214         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1215
1216         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1217                 policy->cur = cpufreq_driver->get(policy->cpu);
1218                 if (!policy->cur) {
1219                         pr_err("%s: ->get() failed\n", __func__);
1220                         goto err_get_freq;
1221                 }
1222         }
1223
1224         /*
1225          * Sometimes boot loaders set CPU frequency to a value outside of
1226          * frequency table present with cpufreq core. In such cases CPU might be
1227          * unstable if it has to run on that frequency for long duration of time
1228          * and so its better to set it to a frequency which is specified in
1229          * freq-table. This also makes cpufreq stats inconsistent as
1230          * cpufreq-stats would fail to register because current frequency of CPU
1231          * isn't found in freq-table.
1232          *
1233          * Because we don't want this change to effect boot process badly, we go
1234          * for the next freq which is >= policy->cur ('cur' must be set by now,
1235          * otherwise we will end up setting freq to lowest of the table as 'cur'
1236          * is initialized to zero).
1237          *
1238          * We are passing target-freq as "policy->cur - 1" otherwise
1239          * __cpufreq_driver_target() would simply fail, as policy->cur will be
1240          * equal to target-freq.
1241          */
1242         if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1243             && has_target()) {
1244                 /* Are we running at unknown frequency ? */
1245                 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1246                 if (ret == -EINVAL) {
1247                         /* Warn user and fix it */
1248                         pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1249                                 __func__, policy->cpu, policy->cur);
1250                         ret = __cpufreq_driver_target(policy, policy->cur - 1,
1251                                 CPUFREQ_RELATION_L);
1252
1253                         /*
1254                          * Reaching here after boot in a few seconds may not
1255                          * mean that system will remain stable at "unknown"
1256                          * frequency for longer duration. Hence, a BUG_ON().
1257                          */
1258                         BUG_ON(ret);
1259                         pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1260                                 __func__, policy->cpu, policy->cur);
1261                 }
1262         }
1263
1264         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1265                                      CPUFREQ_START, policy);
1266
1267         if (!recover_policy) {
1268                 ret = cpufreq_add_dev_interface(policy, dev);
1269                 if (ret)
1270                         goto err_out_unregister;
1271                 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1272                                 CPUFREQ_CREATE_POLICY, policy);
1273         }
1274
1275         write_lock_irqsave(&cpufreq_driver_lock, flags);
1276         list_add(&policy->policy_list, &cpufreq_policy_list);
1277         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1278
1279         cpufreq_init_policy(policy);
1280
1281         if (!recover_policy) {
1282                 policy->user_policy.policy = policy->policy;
1283                 policy->user_policy.governor = policy->governor;
1284         }
1285         up_write(&policy->rwsem);
1286
1287         kobject_uevent(&policy->kobj, KOBJ_ADD);
1288         up_read(&cpufreq_rwsem);
1289
1290         pr_debug("initialization complete\n");
1291
1292         return 0;
1293
1294 err_out_unregister:
1295 err_get_freq:
1296         write_lock_irqsave(&cpufreq_driver_lock, flags);
1297         for_each_cpu(j, policy->cpus)
1298                 per_cpu(cpufreq_cpu_data, j) = NULL;
1299         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1300
1301         if (!recover_policy) {
1302                 kobject_put(&policy->kobj);
1303                 wait_for_completion(&policy->kobj_unregister);
1304         }
1305 err_init_policy_kobj:
1306         up_write(&policy->rwsem);
1307
1308         if (cpufreq_driver->exit)
1309                 cpufreq_driver->exit(policy);
1310 err_set_policy_cpu:
1311         if (recover_policy) {
1312                 /* Do not leave stale fallback data behind. */
1313                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1314                 cpufreq_policy_put_kobj(policy);
1315         }
1316         cpufreq_policy_free(policy);
1317
1318 nomem_out:
1319         up_read(&cpufreq_rwsem);
1320
1321         return ret;
1322 }
1323
1324 /**
1325  * cpufreq_add_dev - add a CPU device
1326  *
1327  * Adds the cpufreq interface for a CPU device.
1328  *
1329  * The Oracle says: try running cpufreq registration/unregistration concurrently
1330  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1331  * mess up, but more thorough testing is needed. - Mathieu
1332  */
1333 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1334 {
1335         return __cpufreq_add_dev(dev, sif);
1336 }
1337
1338 static int __cpufreq_remove_dev_prepare(struct device *dev,
1339                                         struct subsys_interface *sif)
1340 {
1341         unsigned int cpu = dev->id, cpus;
1342         int ret;
1343         unsigned long flags;
1344         struct cpufreq_policy *policy;
1345
1346         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1347
1348         write_lock_irqsave(&cpufreq_driver_lock, flags);
1349
1350         policy = per_cpu(cpufreq_cpu_data, cpu);
1351
1352         /* Save the policy somewhere when doing a light-weight tear-down */
1353         if (cpufreq_suspended)
1354                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1355
1356         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1357
1358         if (!policy) {
1359                 pr_debug("%s: No cpu_data found\n", __func__);
1360                 return -EINVAL;
1361         }
1362
1363         if (has_target()) {
1364                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1365                 if (ret) {
1366                         pr_err("%s: Failed to stop governor\n", __func__);
1367                         return ret;
1368                 }
1369         }
1370
1371         if (!cpufreq_driver->setpolicy)
1372                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1373                         policy->governor->name, CPUFREQ_NAME_LEN);
1374
1375         down_read(&policy->rwsem);
1376         cpus = cpumask_weight(policy->cpus);
1377         up_read(&policy->rwsem);
1378
1379         if (cpu != policy->cpu) {
1380                 sysfs_remove_link(&dev->kobj, "cpufreq");
1381         } else if (cpus > 1) {
1382                 /* Nominate new CPU */
1383                 int new_cpu = cpumask_any_but(policy->cpus, cpu);
1384                 struct device *cpu_dev = get_cpu_device(new_cpu);
1385
1386                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1387                 ret = update_policy_cpu(policy, new_cpu, cpu_dev);
1388                 if (ret) {
1389                         if (sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1390                                               "cpufreq"))
1391                                 pr_err("%s: Failed to restore kobj link to cpu:%d\n",
1392                                        __func__, cpu_dev->id);
1393                         return ret;
1394                 }
1395
1396                 if (!cpufreq_suspended)
1397                         pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1398                                  __func__, new_cpu, cpu);
1399         } else if (cpufreq_driver->stop_cpu) {
1400                 cpufreq_driver->stop_cpu(policy);
1401         }
1402
1403         return 0;
1404 }
1405
1406 static int __cpufreq_remove_dev_finish(struct device *dev,
1407                                        struct subsys_interface *sif)
1408 {
1409         unsigned int cpu = dev->id, cpus;
1410         int ret;
1411         unsigned long flags;
1412         struct cpufreq_policy *policy;
1413
1414         read_lock_irqsave(&cpufreq_driver_lock, flags);
1415         policy = per_cpu(cpufreq_cpu_data, cpu);
1416         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1417
1418         if (!policy) {
1419                 pr_debug("%s: No cpu_data found\n", __func__);
1420                 return -EINVAL;
1421         }
1422
1423         down_write(&policy->rwsem);
1424         cpus = cpumask_weight(policy->cpus);
1425
1426         if (cpus > 1)
1427                 cpumask_clear_cpu(cpu, policy->cpus);
1428         up_write(&policy->rwsem);
1429
1430         /* If cpu is last user of policy, free policy */
1431         if (cpus == 1) {
1432                 if (has_target()) {
1433                         ret = __cpufreq_governor(policy,
1434                                         CPUFREQ_GOV_POLICY_EXIT);
1435                         if (ret) {
1436                                 pr_err("%s: Failed to exit governor\n",
1437                                        __func__);
1438                                 return ret;
1439                         }
1440                 }
1441
1442                 if (!cpufreq_suspended)
1443                         cpufreq_policy_put_kobj(policy);
1444
1445                 /*
1446                  * Perform the ->exit() even during light-weight tear-down,
1447                  * since this is a core component, and is essential for the
1448                  * subsequent light-weight ->init() to succeed.
1449                  */
1450                 if (cpufreq_driver->exit)
1451                         cpufreq_driver->exit(policy);
1452
1453                 /* Remove policy from list of active policies */
1454                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1455                 list_del(&policy->policy_list);
1456                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1457
1458                 if (!cpufreq_suspended)
1459                         cpufreq_policy_free(policy);
1460         } else if (has_target()) {
1461                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1462                 if (!ret)
1463                         ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1464
1465                 if (ret) {
1466                         pr_err("%s: Failed to start governor\n", __func__);
1467                         return ret;
1468                 }
1469         }
1470
1471         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1472         return 0;
1473 }
1474
1475 /**
1476  * cpufreq_remove_dev - remove a CPU device
1477  *
1478  * Removes the cpufreq interface for a CPU device.
1479  */
1480 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1481 {
1482         unsigned int cpu = dev->id;
1483         int ret;
1484
1485         if (cpu_is_offline(cpu))
1486                 return 0;
1487
1488         ret = __cpufreq_remove_dev_prepare(dev, sif);
1489
1490         if (!ret)
1491                 ret = __cpufreq_remove_dev_finish(dev, sif);
1492
1493         return ret;
1494 }
1495
1496 static void handle_update(struct work_struct *work)
1497 {
1498         struct cpufreq_policy *policy =
1499                 container_of(work, struct cpufreq_policy, update);
1500         unsigned int cpu = policy->cpu;
1501         pr_debug("handle_update for cpu %u called\n", cpu);
1502         cpufreq_update_policy(cpu);
1503 }
1504
1505 /**
1506  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1507  *      in deep trouble.
1508  *      @cpu: cpu number
1509  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1510  *      @new_freq: CPU frequency the CPU actually runs at
1511  *
1512  *      We adjust to current frequency first, and need to clean up later.
1513  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1514  */
1515 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1516                                 unsigned int new_freq)
1517 {
1518         struct cpufreq_policy *policy;
1519         struct cpufreq_freqs freqs;
1520         unsigned long flags;
1521
1522         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1523                  old_freq, new_freq);
1524
1525         freqs.old = old_freq;
1526         freqs.new = new_freq;
1527
1528         read_lock_irqsave(&cpufreq_driver_lock, flags);
1529         policy = per_cpu(cpufreq_cpu_data, cpu);
1530         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1531
1532         cpufreq_freq_transition_begin(policy, &freqs);
1533         cpufreq_freq_transition_end(policy, &freqs, 0);
1534 }
1535
1536 /**
1537  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1538  * @cpu: CPU number
1539  *
1540  * This is the last known freq, without actually getting it from the driver.
1541  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1542  */
1543 unsigned int cpufreq_quick_get(unsigned int cpu)
1544 {
1545         struct cpufreq_policy *policy;
1546         unsigned int ret_freq = 0;
1547
1548         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1549                 return cpufreq_driver->get(cpu);
1550
1551         policy = cpufreq_cpu_get(cpu);
1552         if (policy) {
1553                 ret_freq = policy->cur;
1554                 cpufreq_cpu_put(policy);
1555         }
1556
1557         return ret_freq;
1558 }
1559 EXPORT_SYMBOL(cpufreq_quick_get);
1560
1561 /**
1562  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1563  * @cpu: CPU number
1564  *
1565  * Just return the max possible frequency for a given CPU.
1566  */
1567 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1568 {
1569         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1570         unsigned int ret_freq = 0;
1571
1572         if (policy) {
1573                 ret_freq = policy->max;
1574                 cpufreq_cpu_put(policy);
1575         }
1576
1577         return ret_freq;
1578 }
1579 EXPORT_SYMBOL(cpufreq_quick_get_max);
1580
1581 static unsigned int __cpufreq_get(unsigned int cpu)
1582 {
1583         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1584         unsigned int ret_freq = 0;
1585
1586         if (!cpufreq_driver->get)
1587                 return ret_freq;
1588
1589         ret_freq = cpufreq_driver->get(cpu);
1590
1591         if (ret_freq && policy->cur &&
1592                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1593                 /* verify no discrepancy between actual and
1594                                         saved value exists */
1595                 if (unlikely(ret_freq != policy->cur)) {
1596                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1597                         schedule_work(&policy->update);
1598                 }
1599         }
1600
1601         return ret_freq;
1602 }
1603
1604 /**
1605  * cpufreq_get - get the current CPU frequency (in kHz)
1606  * @cpu: CPU number
1607  *
1608  * Get the CPU current (static) CPU frequency
1609  */
1610 unsigned int cpufreq_get(unsigned int cpu)
1611 {
1612         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1613         unsigned int ret_freq = 0;
1614
1615         if (policy) {
1616                 down_read(&policy->rwsem);
1617                 ret_freq = __cpufreq_get(cpu);
1618                 up_read(&policy->rwsem);
1619
1620                 cpufreq_cpu_put(policy);
1621         }
1622
1623         return ret_freq;
1624 }
1625 EXPORT_SYMBOL(cpufreq_get);
1626
1627 static struct subsys_interface cpufreq_interface = {
1628         .name           = "cpufreq",
1629         .subsys         = &cpu_subsys,
1630         .add_dev        = cpufreq_add_dev,
1631         .remove_dev     = cpufreq_remove_dev,
1632 };
1633
1634 /*
1635  * In case platform wants some specific frequency to be configured
1636  * during suspend..
1637  */
1638 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1639 {
1640         int ret;
1641
1642         if (!policy->suspend_freq) {
1643                 pr_err("%s: suspend_freq can't be zero\n", __func__);
1644                 return -EINVAL;
1645         }
1646
1647         pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1648                         policy->suspend_freq);
1649
1650         ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1651                         CPUFREQ_RELATION_H);
1652         if (ret)
1653                 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1654                                 __func__, policy->suspend_freq, ret);
1655
1656         return ret;
1657 }
1658 EXPORT_SYMBOL(cpufreq_generic_suspend);
1659
1660 /**
1661  * cpufreq_suspend() - Suspend CPUFreq governors
1662  *
1663  * Called during system wide Suspend/Hibernate cycles for suspending governors
1664  * as some platforms can't change frequency after this point in suspend cycle.
1665  * Because some of the devices (like: i2c, regulators, etc) they use for
1666  * changing frequency are suspended quickly after this point.
1667  */
1668 void cpufreq_suspend(void)
1669 {
1670         struct cpufreq_policy *policy;
1671
1672         if (!cpufreq_driver)
1673                 return;
1674
1675         if (!has_target())
1676                 goto suspend;
1677
1678         pr_debug("%s: Suspending Governors\n", __func__);
1679
1680         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1681                 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1682                         pr_err("%s: Failed to stop governor for policy: %p\n",
1683                                 __func__, policy);
1684                 else if (cpufreq_driver->suspend
1685                     && cpufreq_driver->suspend(policy))
1686                         pr_err("%s: Failed to suspend driver: %p\n", __func__,
1687                                 policy);
1688         }
1689
1690 suspend:
1691         cpufreq_suspended = true;
1692 }
1693
1694 /**
1695  * cpufreq_resume() - Resume CPUFreq governors
1696  *
1697  * Called during system wide Suspend/Hibernate cycle for resuming governors that
1698  * are suspended with cpufreq_suspend().
1699  */
1700 void cpufreq_resume(void)
1701 {
1702         struct cpufreq_policy *policy;
1703
1704         if (!cpufreq_driver)
1705                 return;
1706
1707         cpufreq_suspended = false;
1708
1709         if (!has_target())
1710                 return;
1711
1712         pr_debug("%s: Resuming Governors\n", __func__);
1713
1714         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1715                 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1716                         pr_err("%s: Failed to resume driver: %p\n", __func__,
1717                                 policy);
1718                 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1719                     || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1720                         pr_err("%s: Failed to start governor for policy: %p\n",
1721                                 __func__, policy);
1722
1723                 /*
1724                  * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1725                  * policy in list. It will verify that the current freq is in
1726                  * sync with what we believe it to be.
1727                  */
1728                 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1729                         schedule_work(&policy->update);
1730         }
1731 }
1732
1733 /**
1734  *      cpufreq_get_current_driver - return current driver's name
1735  *
1736  *      Return the name string of the currently loaded cpufreq driver
1737  *      or NULL, if none.
1738  */
1739 const char *cpufreq_get_current_driver(void)
1740 {
1741         if (cpufreq_driver)
1742                 return cpufreq_driver->name;
1743
1744         return NULL;
1745 }
1746 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1747
1748 /**
1749  *      cpufreq_get_driver_data - return current driver data
1750  *
1751  *      Return the private data of the currently loaded cpufreq
1752  *      driver, or NULL if no cpufreq driver is loaded.
1753  */
1754 void *cpufreq_get_driver_data(void)
1755 {
1756         if (cpufreq_driver)
1757                 return cpufreq_driver->driver_data;
1758
1759         return NULL;
1760 }
1761 EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1762
1763 /*********************************************************************
1764  *                     NOTIFIER LISTS INTERFACE                      *
1765  *********************************************************************/
1766
1767 /**
1768  *      cpufreq_register_notifier - register a driver with cpufreq
1769  *      @nb: notifier function to register
1770  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1771  *
1772  *      Add a driver to one of two lists: either a list of drivers that
1773  *      are notified about clock rate changes (once before and once after
1774  *      the transition), or a list of drivers that are notified about
1775  *      changes in cpufreq policy.
1776  *
1777  *      This function may sleep, and has the same return conditions as
1778  *      blocking_notifier_chain_register.
1779  */
1780 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1781 {
1782         int ret;
1783
1784         if (cpufreq_disabled())
1785                 return -EINVAL;
1786
1787         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1788
1789         switch (list) {
1790         case CPUFREQ_TRANSITION_NOTIFIER:
1791                 ret = srcu_notifier_chain_register(
1792                                 &cpufreq_transition_notifier_list, nb);
1793                 break;
1794         case CPUFREQ_POLICY_NOTIFIER:
1795                 ret = blocking_notifier_chain_register(
1796                                 &cpufreq_policy_notifier_list, nb);
1797                 break;
1798         default:
1799                 ret = -EINVAL;
1800         }
1801
1802         return ret;
1803 }
1804 EXPORT_SYMBOL(cpufreq_register_notifier);
1805
1806 /**
1807  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1808  *      @nb: notifier block to be unregistered
1809  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1810  *
1811  *      Remove a driver from the CPU frequency notifier list.
1812  *
1813  *      This function may sleep, and has the same return conditions as
1814  *      blocking_notifier_chain_unregister.
1815  */
1816 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1817 {
1818         int ret;
1819
1820         if (cpufreq_disabled())
1821                 return -EINVAL;
1822
1823         switch (list) {
1824         case CPUFREQ_TRANSITION_NOTIFIER:
1825                 ret = srcu_notifier_chain_unregister(
1826                                 &cpufreq_transition_notifier_list, nb);
1827                 break;
1828         case CPUFREQ_POLICY_NOTIFIER:
1829                 ret = blocking_notifier_chain_unregister(
1830                                 &cpufreq_policy_notifier_list, nb);
1831                 break;
1832         default:
1833                 ret = -EINVAL;
1834         }
1835
1836         return ret;
1837 }
1838 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1839
1840
1841 /*********************************************************************
1842  *                              GOVERNORS                            *
1843  *********************************************************************/
1844
1845 /* Must set freqs->new to intermediate frequency */
1846 static int __target_intermediate(struct cpufreq_policy *policy,
1847                                  struct cpufreq_freqs *freqs, int index)
1848 {
1849         int ret;
1850
1851         freqs->new = cpufreq_driver->get_intermediate(policy, index);
1852
1853         /* We don't need to switch to intermediate freq */
1854         if (!freqs->new)
1855                 return 0;
1856
1857         pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
1858                  __func__, policy->cpu, freqs->old, freqs->new);
1859
1860         cpufreq_freq_transition_begin(policy, freqs);
1861         ret = cpufreq_driver->target_intermediate(policy, index);
1862         cpufreq_freq_transition_end(policy, freqs, ret);
1863
1864         if (ret)
1865                 pr_err("%s: Failed to change to intermediate frequency: %d\n",
1866                        __func__, ret);
1867
1868         return ret;
1869 }
1870
1871 static int __target_index(struct cpufreq_policy *policy,
1872                           struct cpufreq_frequency_table *freq_table, int index)
1873 {
1874         struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
1875         unsigned int intermediate_freq = 0;
1876         int retval = -EINVAL;
1877         bool notify;
1878
1879         notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1880         if (notify) {
1881                 /* Handle switching to intermediate frequency */
1882                 if (cpufreq_driver->get_intermediate) {
1883                         retval = __target_intermediate(policy, &freqs, index);
1884                         if (retval)
1885                                 return retval;
1886
1887                         intermediate_freq = freqs.new;
1888                         /* Set old freq to intermediate */
1889                         if (intermediate_freq)
1890                                 freqs.old = freqs.new;
1891                 }
1892
1893                 freqs.new = freq_table[index].frequency;
1894                 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1895                          __func__, policy->cpu, freqs.old, freqs.new);
1896
1897                 cpufreq_freq_transition_begin(policy, &freqs);
1898         }
1899
1900         retval = cpufreq_driver->target_index(policy, index);
1901         if (retval)
1902                 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
1903                        retval);
1904
1905         if (notify) {
1906                 cpufreq_freq_transition_end(policy, &freqs, retval);
1907
1908                 /*
1909                  * Failed after setting to intermediate freq? Driver should have
1910                  * reverted back to initial frequency and so should we. Check
1911                  * here for intermediate_freq instead of get_intermediate, in
1912                  * case we have't switched to intermediate freq at all.
1913                  */
1914                 if (unlikely(retval && intermediate_freq)) {
1915                         freqs.old = intermediate_freq;
1916                         freqs.new = policy->restore_freq;
1917                         cpufreq_freq_transition_begin(policy, &freqs);
1918                         cpufreq_freq_transition_end(policy, &freqs, 0);
1919                 }
1920         }
1921
1922         return retval;
1923 }
1924
1925 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1926                             unsigned int target_freq,
1927                             unsigned int relation)
1928 {
1929         unsigned int old_target_freq = target_freq;
1930         int retval = -EINVAL;
1931
1932         if (cpufreq_disabled())
1933                 return -ENODEV;
1934
1935         /* Make sure that target_freq is within supported range */
1936         if (target_freq > policy->max)
1937                 target_freq = policy->max;
1938         if (target_freq < policy->min)
1939                 target_freq = policy->min;
1940
1941         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1942                  policy->cpu, target_freq, relation, old_target_freq);
1943
1944         /*
1945          * This might look like a redundant call as we are checking it again
1946          * after finding index. But it is left intentionally for cases where
1947          * exactly same freq is called again and so we can save on few function
1948          * calls.
1949          */
1950         if (target_freq == policy->cur)
1951                 return 0;
1952
1953         /* Save last value to restore later on errors */
1954         policy->restore_freq = policy->cur;
1955
1956         if (cpufreq_driver->target)
1957                 retval = cpufreq_driver->target(policy, target_freq, relation);
1958         else if (cpufreq_driver->target_index) {
1959                 struct cpufreq_frequency_table *freq_table;
1960                 int index;
1961
1962                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1963                 if (unlikely(!freq_table)) {
1964                         pr_err("%s: Unable to find freq_table\n", __func__);
1965                         goto out;
1966                 }
1967
1968                 retval = cpufreq_frequency_table_target(policy, freq_table,
1969                                 target_freq, relation, &index);
1970                 if (unlikely(retval)) {
1971                         pr_err("%s: Unable to find matching freq\n", __func__);
1972                         goto out;
1973                 }
1974
1975                 if (freq_table[index].frequency == policy->cur) {
1976                         retval = 0;
1977                         goto out;
1978                 }
1979
1980                 retval = __target_index(policy, freq_table, index);
1981         }
1982
1983 out:
1984         return retval;
1985 }
1986 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1987
1988 int cpufreq_driver_target(struct cpufreq_policy *policy,
1989                           unsigned int target_freq,
1990                           unsigned int relation)
1991 {
1992         int ret = -EINVAL;
1993
1994         down_write(&policy->rwsem);
1995
1996         ret = __cpufreq_driver_target(policy, target_freq, relation);
1997
1998         up_write(&policy->rwsem);
1999
2000         return ret;
2001 }
2002 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2003
2004 /*
2005  * when "event" is CPUFREQ_GOV_LIMITS
2006  */
2007
2008 static int __cpufreq_governor(struct cpufreq_policy *policy,
2009                                         unsigned int event)
2010 {
2011         int ret;
2012
2013         /* Only must be defined when default governor is known to have latency
2014            restrictions, like e.g. conservative or ondemand.
2015            That this is the case is already ensured in Kconfig
2016         */
2017 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
2018         struct cpufreq_governor *gov = &cpufreq_gov_performance;
2019 #else
2020         struct cpufreq_governor *gov = NULL;
2021 #endif
2022
2023         /* Don't start any governor operations if we are entering suspend */
2024         if (cpufreq_suspended)
2025                 return 0;
2026
2027         if (policy->governor->max_transition_latency &&
2028             policy->cpuinfo.transition_latency >
2029             policy->governor->max_transition_latency) {
2030                 if (!gov)
2031                         return -EINVAL;
2032                 else {
2033                         pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
2034                                 policy->governor->name, gov->name);
2035                         policy->governor = gov;
2036                 }
2037         }
2038
2039         if (event == CPUFREQ_GOV_POLICY_INIT)
2040                 if (!try_module_get(policy->governor->owner))
2041                         return -EINVAL;
2042
2043         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
2044                  policy->cpu, event);
2045
2046         mutex_lock(&cpufreq_governor_lock);
2047         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
2048             || (!policy->governor_enabled
2049             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
2050                 mutex_unlock(&cpufreq_governor_lock);
2051                 return -EBUSY;
2052         }
2053
2054         if (event == CPUFREQ_GOV_STOP)
2055                 policy->governor_enabled = false;
2056         else if (event == CPUFREQ_GOV_START)
2057                 policy->governor_enabled = true;
2058
2059         mutex_unlock(&cpufreq_governor_lock);
2060
2061         ret = policy->governor->governor(policy, event);
2062
2063         if (!ret) {
2064                 if (event == CPUFREQ_GOV_POLICY_INIT)
2065                         policy->governor->initialized++;
2066                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
2067                         policy->governor->initialized--;
2068         } else {
2069                 /* Restore original values */
2070                 mutex_lock(&cpufreq_governor_lock);
2071                 if (event == CPUFREQ_GOV_STOP)
2072                         policy->governor_enabled = true;
2073                 else if (event == CPUFREQ_GOV_START)
2074                         policy->governor_enabled = false;
2075                 mutex_unlock(&cpufreq_governor_lock);
2076         }
2077
2078         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
2079                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
2080                 module_put(policy->governor->owner);
2081
2082         return ret;
2083 }
2084
2085 int cpufreq_register_governor(struct cpufreq_governor *governor)
2086 {
2087         int err;
2088
2089         if (!governor)
2090                 return -EINVAL;
2091
2092         if (cpufreq_disabled())
2093                 return -ENODEV;
2094
2095         mutex_lock(&cpufreq_governor_mutex);
2096
2097         governor->initialized = 0;
2098         err = -EBUSY;
2099         if (__find_governor(governor->name) == NULL) {
2100                 err = 0;
2101                 list_add(&governor->governor_list, &cpufreq_governor_list);
2102         }
2103
2104         mutex_unlock(&cpufreq_governor_mutex);
2105         return err;
2106 }
2107 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2108
2109 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2110 {
2111         int cpu;
2112
2113         if (!governor)
2114                 return;
2115
2116         if (cpufreq_disabled())
2117                 return;
2118
2119         for_each_present_cpu(cpu) {
2120                 if (cpu_online(cpu))
2121                         continue;
2122                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2123                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2124         }
2125
2126         mutex_lock(&cpufreq_governor_mutex);
2127         list_del(&governor->governor_list);
2128         mutex_unlock(&cpufreq_governor_mutex);
2129         return;
2130 }
2131 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2132
2133
2134 /*********************************************************************
2135  *                          POLICY INTERFACE                         *
2136  *********************************************************************/
2137
2138 /**
2139  * cpufreq_get_policy - get the current cpufreq_policy
2140  * @policy: struct cpufreq_policy into which the current cpufreq_policy
2141  *      is written
2142  *
2143  * Reads the current cpufreq policy.
2144  */
2145 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2146 {
2147         struct cpufreq_policy *cpu_policy;
2148         if (!policy)
2149                 return -EINVAL;
2150
2151         cpu_policy = cpufreq_cpu_get(cpu);
2152         if (!cpu_policy)
2153                 return -EINVAL;
2154
2155         memcpy(policy, cpu_policy, sizeof(*policy));
2156
2157         cpufreq_cpu_put(cpu_policy);
2158         return 0;
2159 }
2160 EXPORT_SYMBOL(cpufreq_get_policy);
2161
2162 /*
2163  * policy : current policy.
2164  * new_policy: policy to be set.
2165  */
2166 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2167                                 struct cpufreq_policy *new_policy)
2168 {
2169         struct cpufreq_governor *old_gov;
2170         int ret;
2171
2172         pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2173                  new_policy->cpu, new_policy->min, new_policy->max);
2174
2175         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2176
2177         if (new_policy->min > policy->max || new_policy->max < policy->min)
2178                 return -EINVAL;
2179
2180         /* verify the cpu speed can be set within this limit */
2181         ret = cpufreq_driver->verify(new_policy);
2182         if (ret)
2183                 return ret;
2184
2185         /* adjust if necessary - all reasons */
2186         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2187                         CPUFREQ_ADJUST, new_policy);
2188
2189         /* adjust if necessary - hardware incompatibility*/
2190         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2191                         CPUFREQ_INCOMPATIBLE, new_policy);
2192
2193         /*
2194          * verify the cpu speed can be set within this limit, which might be
2195          * different to the first one
2196          */
2197         ret = cpufreq_driver->verify(new_policy);
2198         if (ret)
2199                 return ret;
2200
2201         /* notification of the new policy */
2202         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2203                         CPUFREQ_NOTIFY, new_policy);
2204
2205         policy->min = new_policy->min;
2206         policy->max = new_policy->max;
2207
2208         pr_debug("new min and max freqs are %u - %u kHz\n",
2209                  policy->min, policy->max);
2210
2211         if (cpufreq_driver->setpolicy) {
2212                 policy->policy = new_policy->policy;
2213                 pr_debug("setting range\n");
2214                 return cpufreq_driver->setpolicy(new_policy);
2215         }
2216
2217         if (new_policy->governor == policy->governor)
2218                 goto out;
2219
2220         pr_debug("governor switch\n");
2221
2222         /* save old, working values */
2223         old_gov = policy->governor;
2224         /* end old governor */
2225         if (old_gov) {
2226                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2227                 up_write(&policy->rwsem);
2228                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2229                 down_write(&policy->rwsem);
2230         }
2231
2232         /* start new governor */
2233         policy->governor = new_policy->governor;
2234         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2235                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2236                         goto out;
2237
2238                 up_write(&policy->rwsem);
2239                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2240                 down_write(&policy->rwsem);
2241         }
2242
2243         /* new governor failed, so re-start old one */
2244         pr_debug("starting governor %s failed\n", policy->governor->name);
2245         if (old_gov) {
2246                 policy->governor = old_gov;
2247                 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2248                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2249         }
2250
2251         return -EINVAL;
2252
2253  out:
2254         pr_debug("governor: change or update limits\n");
2255         return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2256 }
2257
2258 /**
2259  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2260  *      @cpu: CPU which shall be re-evaluated
2261  *
2262  *      Useful for policy notifiers which have different necessities
2263  *      at different times.
2264  */
2265 int cpufreq_update_policy(unsigned int cpu)
2266 {
2267         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2268         struct cpufreq_policy new_policy;
2269         int ret;
2270
2271         if (!policy)
2272                 return -ENODEV;
2273
2274         down_write(&policy->rwsem);
2275
2276         pr_debug("updating policy for CPU %u\n", cpu);
2277         memcpy(&new_policy, policy, sizeof(*policy));
2278         new_policy.min = policy->user_policy.min;
2279         new_policy.max = policy->user_policy.max;
2280         new_policy.policy = policy->user_policy.policy;
2281         new_policy.governor = policy->user_policy.governor;
2282
2283         /*
2284          * BIOS might change freq behind our back
2285          * -> ask driver for current freq and notify governors about a change
2286          */
2287         if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2288                 new_policy.cur = cpufreq_driver->get(cpu);
2289                 if (WARN_ON(!new_policy.cur)) {
2290                         ret = -EIO;
2291                         goto unlock;
2292                 }
2293
2294                 if (!policy->cur) {
2295                         pr_debug("Driver did not initialize current freq\n");
2296                         policy->cur = new_policy.cur;
2297                 } else {
2298                         if (policy->cur != new_policy.cur && has_target())
2299                                 cpufreq_out_of_sync(cpu, policy->cur,
2300                                                                 new_policy.cur);
2301                 }
2302         }
2303
2304         ret = cpufreq_set_policy(policy, &new_policy);
2305
2306 unlock:
2307         up_write(&policy->rwsem);
2308
2309         cpufreq_cpu_put(policy);
2310         return ret;
2311 }
2312 EXPORT_SYMBOL(cpufreq_update_policy);
2313
2314 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2315                                         unsigned long action, void *hcpu)
2316 {
2317         unsigned int cpu = (unsigned long)hcpu;
2318         struct device *dev;
2319
2320         dev = get_cpu_device(cpu);
2321         if (dev) {
2322                 switch (action & ~CPU_TASKS_FROZEN) {
2323                 case CPU_ONLINE:
2324                         __cpufreq_add_dev(dev, NULL);
2325                         break;
2326
2327                 case CPU_DOWN_PREPARE:
2328                         __cpufreq_remove_dev_prepare(dev, NULL);
2329                         break;
2330
2331                 case CPU_POST_DEAD:
2332                         __cpufreq_remove_dev_finish(dev, NULL);
2333                         break;
2334
2335                 case CPU_DOWN_FAILED:
2336                         __cpufreq_add_dev(dev, NULL);
2337                         break;
2338                 }
2339         }
2340         return NOTIFY_OK;
2341 }
2342
2343 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2344         .notifier_call = cpufreq_cpu_callback,
2345 };
2346
2347 /*********************************************************************
2348  *               BOOST                                               *
2349  *********************************************************************/
2350 static int cpufreq_boost_set_sw(int state)
2351 {
2352         struct cpufreq_frequency_table *freq_table;
2353         struct cpufreq_policy *policy;
2354         int ret = -EINVAL;
2355
2356         list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2357                 freq_table = cpufreq_frequency_get_table(policy->cpu);
2358                 if (freq_table) {
2359                         ret = cpufreq_frequency_table_cpuinfo(policy,
2360                                                         freq_table);
2361                         if (ret) {
2362                                 pr_err("%s: Policy frequency update failed\n",
2363                                        __func__);
2364                                 break;
2365                         }
2366                         policy->user_policy.max = policy->max;
2367                         __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2368                 }
2369         }
2370
2371         return ret;
2372 }
2373
2374 int cpufreq_boost_trigger_state(int state)
2375 {
2376         unsigned long flags;
2377         int ret = 0;
2378
2379         if (cpufreq_driver->boost_enabled == state)
2380                 return 0;
2381
2382         write_lock_irqsave(&cpufreq_driver_lock, flags);
2383         cpufreq_driver->boost_enabled = state;
2384         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2385
2386         ret = cpufreq_driver->set_boost(state);
2387         if (ret) {
2388                 write_lock_irqsave(&cpufreq_driver_lock, flags);
2389                 cpufreq_driver->boost_enabled = !state;
2390                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2391
2392                 pr_err("%s: Cannot %s BOOST\n",
2393                        __func__, state ? "enable" : "disable");
2394         }
2395
2396         return ret;
2397 }
2398
2399 int cpufreq_boost_supported(void)
2400 {
2401         if (likely(cpufreq_driver))
2402                 return cpufreq_driver->boost_supported;
2403
2404         return 0;
2405 }
2406 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2407
2408 int cpufreq_boost_enabled(void)
2409 {
2410         return cpufreq_driver->boost_enabled;
2411 }
2412 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2413
2414 /*********************************************************************
2415  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2416  *********************************************************************/
2417
2418 /**
2419  * cpufreq_register_driver - register a CPU Frequency driver
2420  * @driver_data: A struct cpufreq_driver containing the values#
2421  * submitted by the CPU Frequency driver.
2422  *
2423  * Registers a CPU Frequency driver to this core code. This code
2424  * returns zero on success, -EBUSY when another driver got here first
2425  * (and isn't unregistered in the meantime).
2426  *
2427  */
2428 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2429 {
2430         unsigned long flags;
2431         int ret;
2432
2433         if (cpufreq_disabled())
2434                 return -ENODEV;
2435
2436         if (!driver_data || !driver_data->verify || !driver_data->init ||
2437             !(driver_data->setpolicy || driver_data->target_index ||
2438                     driver_data->target) ||
2439              (driver_data->setpolicy && (driver_data->target_index ||
2440                     driver_data->target)) ||
2441              (!!driver_data->get_intermediate != !!driver_data->target_intermediate))
2442                 return -EINVAL;
2443
2444         pr_debug("trying to register driver %s\n", driver_data->name);
2445
2446         if (driver_data->setpolicy)
2447                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2448
2449         write_lock_irqsave(&cpufreq_driver_lock, flags);
2450         if (cpufreq_driver) {
2451                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2452                 return -EEXIST;
2453         }
2454         cpufreq_driver = driver_data;
2455         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2456
2457         if (cpufreq_boost_supported()) {
2458                 /*
2459                  * Check if driver provides function to enable boost -
2460                  * if not, use cpufreq_boost_set_sw as default
2461                  */
2462                 if (!cpufreq_driver->set_boost)
2463                         cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2464
2465                 ret = cpufreq_sysfs_create_file(&boost.attr);
2466                 if (ret) {
2467                         pr_err("%s: cannot register global BOOST sysfs file\n",
2468                                __func__);
2469                         goto err_null_driver;
2470                 }
2471         }
2472
2473         ret = subsys_interface_register(&cpufreq_interface);
2474         if (ret)
2475                 goto err_boost_unreg;
2476
2477         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2478                 int i;
2479                 ret = -ENODEV;
2480
2481                 /* check for at least one working CPU */
2482                 for (i = 0; i < nr_cpu_ids; i++)
2483                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2484                                 ret = 0;
2485                                 break;
2486                         }
2487
2488                 /* if all ->init() calls failed, unregister */
2489                 if (ret) {
2490                         pr_debug("no CPU initialized for driver %s\n",
2491                                  driver_data->name);
2492                         goto err_if_unreg;
2493                 }
2494         }
2495
2496         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2497         pr_debug("driver %s up and running\n", driver_data->name);
2498
2499         return 0;
2500 err_if_unreg:
2501         subsys_interface_unregister(&cpufreq_interface);
2502 err_boost_unreg:
2503         if (cpufreq_boost_supported())
2504                 cpufreq_sysfs_remove_file(&boost.attr);
2505 err_null_driver:
2506         write_lock_irqsave(&cpufreq_driver_lock, flags);
2507         cpufreq_driver = NULL;
2508         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2509         return ret;
2510 }
2511 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2512
2513 /**
2514  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2515  *
2516  * Unregister the current CPUFreq driver. Only call this if you have
2517  * the right to do so, i.e. if you have succeeded in initialising before!
2518  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2519  * currently not initialised.
2520  */
2521 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2522 {
2523         unsigned long flags;
2524
2525         if (!cpufreq_driver || (driver != cpufreq_driver))
2526                 return -EINVAL;
2527
2528         pr_debug("unregistering driver %s\n", driver->name);
2529
2530         subsys_interface_unregister(&cpufreq_interface);
2531         if (cpufreq_boost_supported())
2532                 cpufreq_sysfs_remove_file(&boost.attr);
2533
2534         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2535
2536         down_write(&cpufreq_rwsem);
2537         write_lock_irqsave(&cpufreq_driver_lock, flags);
2538
2539         cpufreq_driver = NULL;
2540
2541         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2542         up_write(&cpufreq_rwsem);
2543
2544         return 0;
2545 }
2546 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2547
2548 static int __init cpufreq_core_init(void)
2549 {
2550         if (cpufreq_disabled())
2551                 return -ENODEV;
2552
2553         cpufreq_global_kobject = kobject_create();
2554         BUG_ON(!cpufreq_global_kobject);
2555
2556         return 0;
2557 }
2558 core_initcall(cpufreq_core_init);