drivers/cpufreq/intel_pstate.c

   1 /*
   2  * intel_pstate.c: Native P state management for Intel processors
   3  *
   4  * (C) Copyright 2012 Intel Corporation
   5  * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License
   9  * as published by the Free Software Foundation; version 2
  10  * of the License.
  11  */
  12
  13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  14
  15 #include <linux/kernel.h>
  16 #include <linux/kernel_stat.h>
  17 #include <linux/module.h>
  18 #include <linux/ktime.h>
  19 #include <linux/hrtimer.h>
  20 #include <linux/tick.h>
  21 #include <linux/slab.h>
  22 #include <linux/sched/cpufreq.h>
  23 #include <linux/list.h>
  24 #include <linux/cpu.h>
  25 #include <linux/cpufreq.h>
  26 #include <linux/sysfs.h>
  27 #include <linux/types.h>
  28 #include <linux/fs.h>
  29 #include <linux/debugfs.h>
  30 #include <linux/acpi.h>
  31 #include <linux/vmalloc.h>
  32 #include <trace/events/power.h>
  33
  34 #include <asm/div64.h>
  35 #include <asm/msr.h>
  36 #include <asm/cpu_device_id.h>
  37 #include <asm/cpufeature.h>
  38 #include <asm/intel-family.h>
  39
  40 #define INTEL_CPUFREQ_TRANSITION_LATENCY        20000
  41
  42 #ifdef CONFIG_ACPI
  43 #include <acpi/processor.h>
  44 #include <acpi/cppc_acpi.h>
  45 #endif
  46
  47 #define FRAC_BITS 8
  48 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
  49 #define fp_toint(X) ((X) >> FRAC_BITS)
  50
  51 #define EXT_BITS 6
  52 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
  53 #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
  54 #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
  55
  56 static inline int32_t mul_fp(int32_t x, int32_t y)
  57 {
  58         return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
  59 }
  60
  61 static inline int32_t div_fp(s64 x, s64 y)
  62 {
  63         return div64_s64((int64_t)x << FRAC_BITS, y);
  64 }
  65
  66 static inline int ceiling_fp(int32_t x)
  67 {
  68         int mask, ret;
  69
  70         ret = fp_toint(x);
  71         mask = (1 << FRAC_BITS) - 1;
  72         if (x & mask)
  73                 ret += 1;
  74         return ret;
  75 }
  76
  77 static inline u64 mul_ext_fp(u64 x, u64 y)
  78 {
  79         return (x * y) >> EXT_FRAC_BITS;
  80 }
  81
  82 static inline u64 div_ext_fp(u64 x, u64 y)
  83 {
  84         return div64_u64(x << EXT_FRAC_BITS, y);
  85 }
  86
  87 static inline int32_t percent_ext_fp(int percent)
  88 {
  89         return div_ext_fp(percent, 100);
  90 }
  91
  92 /**
  93  * struct sample -      Store performance sample
  94  * @core_avg_perf:      Ratio of APERF/MPERF which is the actual average
  95  *                      performance during last sample period
  96  * @busy_scaled:        Scaled busy value which is used to calculate next
  97  *                      P state. This can be different than core_avg_perf
  98  *                      to account for cpu idle period
  99  * @aperf:              Difference of actual performance frequency clock count
 100  *                      read from APERF MSR between last and current sample
 101  * @mperf:              Difference of maximum performance frequency clock count
 102  *                      read from MPERF MSR between last and current sample
 103  * @tsc:                Difference of time stamp counter between last and
 104  *                      current sample
 105  * @time:               Current time from scheduler
 106  *
 107  * This structure is used in the cpudata structure to store performance sample
 108  * data for choosing next P State.
 109  */
 110 struct sample {
 111         int32_t core_avg_perf;
 112         int32_t busy_scaled;
 113         u64 aperf;
 114         u64 mperf;
 115         u64 tsc;
 116         u64 time;
 117 };
 118
 119 /**
 120  * struct pstate_data - Store P state data
 121  * @current_pstate:     Current requested P state
 122  * @min_pstate:         Min P state possible for this platform
 123  * @max_pstate:         Max P state possible for this platform
 124  * @max_pstate_physical:This is physical Max P state for a processor
 125  *                      This can be higher than the max_pstate which can
 126  *                      be limited by platform thermal design power limits
 127  * @scaling:            Scaling factor to  convert frequency to cpufreq
 128  *                      frequency units
 129  * @turbo_pstate:       Max Turbo P state possible for this platform
 130  * @max_freq:           @max_pstate frequency in cpufreq units
 131  * @turbo_freq:         @turbo_pstate frequency in cpufreq units
 132  *
 133  * Stores the per cpu model P state limits and current P state.
 134  */
 135 struct pstate_data {
 136         int     current_pstate;
 137         int     min_pstate;
 138         int     max_pstate;
 139         int     max_pstate_physical;
 140         int     scaling;
 141         int     turbo_pstate;
 142         unsigned int max_freq;
 143         unsigned int turbo_freq;
 144 };
 145
 146 /**
 147  * struct vid_data -    Stores voltage information data
 148  * @min:                VID data for this platform corresponding to
 149  *                      the lowest P state
 150  * @max:                VID data corresponding to the highest P State.
 151  * @turbo:              VID data for turbo P state
 152  * @ratio:              Ratio of (vid max - vid min) /
 153  *                      (max P state - Min P State)
 154  *
 155  * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
 156  * This data is used in Atom platforms, where in addition to target P state,
 157  * the voltage data needs to be specified to select next P State.
 158  */
 159 struct vid_data {
 160         int min;
 161         int max;
 162         int turbo;
 163         int32_t ratio;
 164 };
 165
 166 /**
 167  * struct _pid -        Stores PID data
 168  * @setpoint:           Target set point for busyness or performance
 169  * @integral:           Storage for accumulated error values
 170  * @p_gain:             PID proportional gain
 171  * @i_gain:             PID integral gain
 172  * @d_gain:             PID derivative gain
 173  * @deadband:           PID deadband
 174  * @last_err:           Last error storage for integral part of PID calculation
 175  *
 176  * Stores PID coefficients and last error for PID controller.
 177  */
 178 struct _pid {
 179         int setpoint;
 180         int32_t integral;
 181         int32_t p_gain;
 182         int32_t i_gain;
 183         int32_t d_gain;
 184         int deadband;
 185         int32_t last_err;
 186 };
 187
 188 /**
 189  * struct perf_limits - Store user and policy limits
 190  * @no_turbo:           User requested turbo state from intel_pstate sysfs
 191  * @turbo_disabled:     Platform turbo status either from msr
 192  *                      MSR_IA32_MISC_ENABLE or when maximum available pstate
 193  *                      matches the maximum turbo pstate
 194  * @max_perf_pct:       Effective maximum performance limit in percentage, this
 195  *                      is minimum of either limits enforced by cpufreq policy
 196  *                      or limits from user set limits via intel_pstate sysfs
 197  * @min_perf_pct:       Effective minimum performance limit in percentage, this
 198  *                      is maximum of either limits enforced by cpufreq policy
 199  *                      or limits from user set limits via intel_pstate sysfs
 200  * @max_perf:           This is a scaled value between 0 to 255 for max_perf_pct
 201  *                      This value is used to limit max pstate
 202  * @min_perf:           This is a scaled value between 0 to 255 for min_perf_pct
 203  *                      This value is used to limit min pstate
 204  * @max_policy_pct:     The maximum performance in percentage enforced by
 205  *                      cpufreq setpolicy interface
 206  * @max_sysfs_pct:      The maximum performance in percentage enforced by
 207  *                      intel pstate sysfs interface, unused when per cpu
 208  *                      controls are enforced
 209  * @min_policy_pct:     The minimum performance in percentage enforced by
 210  *                      cpufreq setpolicy interface
 211  * @min_sysfs_pct:      The minimum performance in percentage enforced by
 212  *                      intel pstate sysfs interface, unused when per cpu
 213  *                      controls are enforced
 214  *
 215  * Storage for user and policy defined limits.
 216  */
 217 struct perf_limits {
 218         int no_turbo;
 219         int turbo_disabled;
 220         int max_perf_pct;
 221         int min_perf_pct;
 222         int32_t max_perf;
 223         int32_t min_perf;
 224         int max_policy_pct;
 225         int max_sysfs_pct;
 226         int min_policy_pct;
 227         int min_sysfs_pct;
 228 };
 229
 230 /**
 231  * struct cpudata -     Per CPU instance data storage
 232  * @cpu:                CPU number for this instance data
 233  * @policy:             CPUFreq policy value
 234  * @update_util:        CPUFreq utility callback information
 235  * @update_util_set:    CPUFreq utility callback is set
 236  * @iowait_boost:       iowait-related boost fraction
 237  * @last_update:        Time of the last update.
 238  * @pstate:             Stores P state limits for this CPU
 239  * @vid:                Stores VID limits for this CPU
 240  * @pid:                Stores PID parameters for this CPU
 241  * @last_sample_time:   Last Sample time
 242  * @prev_aperf:         Last APERF value read from APERF MSR
 243  * @prev_mperf:         Last MPERF value read from MPERF MSR
 244  * @prev_tsc:           Last timestamp counter (TSC) value
 245  * @prev_cummulative_iowait: IO Wait time difference from last and
 246  *                      current sample
 247  * @sample:             Storage for storing last Sample data
 248  * @perf_limits:        Pointer to perf_limit unique to this CPU
 249  *                      Not all field in the structure are applicable
 250  *                      when per cpu controls are enforced
 251  * @acpi_perf_data:     Stores ACPI perf information read from _PSS
 252  * @valid_pss_table:    Set to true for valid ACPI _PSS entries found
 253  * @epp_powersave:      Last saved HWP energy performance preference
 254  *                      (EPP) or energy performance bias (EPB),
 255  *                      when policy switched to performance
 256  * @epp_policy:         Last saved policy used to set EPP/EPB
 257  * @epp_default:        Power on default HWP energy performance
 258  *                      preference/bias
 259  * @epp_saved:          Saved EPP/EPB during system suspend or CPU offline
 260  *                      operation
 261  *
 262  * This structure stores per CPU instance data for all CPUs.
 263  */
 264 struct cpudata {
 265         int cpu;
 266
 267         unsigned int policy;
 268         struct update_util_data update_util;
 269         bool   update_util_set;
 270
 271         struct pstate_data pstate;
 272         struct vid_data vid;
 273         struct _pid pid;
 274
 275         u64     last_update;
 276         u64     last_sample_time;
 277         u64     prev_aperf;
 278         u64     prev_mperf;
 279         u64     prev_tsc;
 280         u64     prev_cummulative_iowait;
 281         struct sample sample;
 282         struct perf_limits *perf_limits;
 283 #ifdef CONFIG_ACPI
 284         struct acpi_processor_performance acpi_perf_data;
 285         bool valid_pss_table;
 286 #endif
 287         unsigned int iowait_boost;
 288         s16 epp_powersave;
 289         s16 epp_policy;
 290         s16 epp_default;
 291         s16 epp_saved;
 292 };
 293
 294 static struct cpudata **all_cpu_data;
 295
 296 /**
 297  * struct pstate_adjust_policy - Stores static PID configuration data
 298  * @sample_rate_ms:     PID calculation sample rate in ms
 299  * @sample_rate_ns:     Sample rate calculation in ns
 300  * @deadband:           PID deadband
 301  * @setpoint:           PID Setpoint
 302  * @p_gain_pct:         PID proportional gain
 303  * @i_gain_pct:         PID integral gain
 304  * @d_gain_pct:         PID derivative gain
 305  *
 306  * Stores per CPU model static PID configuration data.
 307  */
 308 struct pstate_adjust_policy {
 309         int sample_rate_ms;
 310         s64 sample_rate_ns;
 311         int deadband;
 312         int setpoint;
 313         int p_gain_pct;
 314         int d_gain_pct;
 315         int i_gain_pct;
 316 };
 317
 318 /**
 319  * struct pstate_funcs - Per CPU model specific callbacks
 320  * @get_max:            Callback to get maximum non turbo effective P state
 321  * @get_max_physical:   Callback to get maximum non turbo physical P state
 322  * @get_min:            Callback to get minimum P state
 323  * @get_turbo:          Callback to get turbo P state
 324  * @get_scaling:        Callback to get frequency scaling factor
 325  * @get_val:            Callback to convert P state to actual MSR write value
 326  * @get_vid:            Callback to get VID data for Atom platforms
 327  * @get_target_pstate:  Callback to a function to calculate next P state to use
 328  *
 329  * Core and Atom CPU models have different way to get P State limits. This
 330  * structure is used to store those callbacks.
 331  */
 332 struct pstate_funcs {
 333         int (*get_max)(void);
 334         int (*get_max_physical)(void);
 335         int (*get_min)(void);
 336         int (*get_turbo)(void);
 337         int (*get_scaling)(void);
 338         u64 (*get_val)(struct cpudata*, int pstate);
 339         void (*get_vid)(struct cpudata *);
 340         int32_t (*get_target_pstate)(struct cpudata *);
 341 };
 342
 343 /**
 344  * struct cpu_defaults- Per CPU model default config data
 345  * @pid_policy: PID config data
 346  * @funcs:              Callback function data
 347  */
 348 struct cpu_defaults {
 349         struct pstate_adjust_policy pid_policy;
 350         struct pstate_funcs funcs;
 351 };
 352
 353 static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu);
 354 static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu);
 355
 356 static struct pstate_adjust_policy pid_params __read_mostly;
 357 static struct pstate_funcs pstate_funcs __read_mostly;
 358 static int hwp_active __read_mostly;
 359 static bool per_cpu_limits __read_mostly;
 360
 361 static bool driver_registered __read_mostly;
 362
 363 #ifdef CONFIG_ACPI
 364 static bool acpi_ppc;
 365 #endif
 366
 367 static struct perf_limits global;
 368
 369 static void intel_pstate_init_limits(struct perf_limits *limits)
 370 {
 371         memset(limits, 0, sizeof(*limits));
 372         limits->max_perf_pct = 100;
 373         limits->max_perf = int_ext_tofp(1);
 374         limits->max_policy_pct = 100;
 375         limits->max_sysfs_pct = 100;
 376 }
 377
 378 static DEFINE_MUTEX(intel_pstate_driver_lock);
 379 static DEFINE_MUTEX(intel_pstate_limits_lock);
 380
 381 #ifdef CONFIG_ACPI
 382
 383 static bool intel_pstate_get_ppc_enable_status(void)
 384 {
 385         if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
 386             acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
 387                 return true;
 388
 389         return acpi_ppc;
 390 }
 391
 392 #ifdef CONFIG_ACPI_CPPC_LIB
 393
 394 /* The work item is needed to avoid CPU hotplug locking issues */
 395 static void intel_pstste_sched_itmt_work_fn(struct work_struct *work)
 396 {
 397         sched_set_itmt_support();
 398 }
 399
 400 static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn);
 401
 402 static void intel_pstate_set_itmt_prio(int cpu)
 403 {
 404         struct cppc_perf_caps cppc_perf;
 405         static u32 max_highest_perf = 0, min_highest_perf = U32_MAX;
 406         int ret;
 407
 408         ret = cppc_get_perf_caps(cpu, &cppc_perf);
 409         if (ret)
 410                 return;
 411
 412         /*
 413          * The priorities can be set regardless of whether or not
 414          * sched_set_itmt_support(true) has been called and it is valid to
 415          * update them at any time after it has been called.
 416          */
 417         sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);
 418
 419         if (max_highest_perf <= min_highest_perf) {
 420                 if (cppc_perf.highest_perf > max_highest_perf)
 421                         max_highest_perf = cppc_perf.highest_perf;
 422
 423                 if (cppc_perf.highest_perf < min_highest_perf)
 424                         min_highest_perf = cppc_perf.highest_perf;
 425
 426                 if (max_highest_perf > min_highest_perf) {
 427                         /*
 428                          * This code can be run during CPU online under the
 429                          * CPU hotplug locks, so sched_set_itmt_support()
 430                          * cannot be called from here.  Queue up a work item
 431                          * to invoke it.
 432                          */
 433                         schedule_work(&sched_itmt_work);
 434                 }
 435         }
 436 }
 437 #else
 438 static void intel_pstate_set_itmt_prio(int cpu)
 439 {
 440 }
 441 #endif
 442
 443 static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
 444 {
 445         struct cpudata *cpu;
 446         int ret;
 447         int i;
 448
 449         if (hwp_active) {
 450                 intel_pstate_set_itmt_prio(policy->cpu);
 451                 return;
 452         }
 453
 454         if (!intel_pstate_get_ppc_enable_status())
 455                 return;
 456
 457         cpu = all_cpu_data[policy->cpu];
 458
 459         ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
 460                                                   policy->cpu);
 461         if (ret)
 462                 return;
 463
 464         /*
 465          * Check if the control value in _PSS is for PERF_CTL MSR, which should
 466          * guarantee that the states returned by it map to the states in our
 467          * list directly.
 468          */
 469         if (cpu->acpi_perf_data.control_register.space_id !=
 470                                                 ACPI_ADR_SPACE_FIXED_HARDWARE)
 471                 goto err;
 472
 473         /*
 474          * If there is only one entry _PSS, simply ignore _PSS and continue as
 475          * usual without taking _PSS into account
 476          */
 477         if (cpu->acpi_perf_data.state_count < 2)
 478                 goto err;
 479
 480         pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
 481         for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
 482                 pr_debug("     %cP%d: %u MHz, %u mW, 0x%x\n",
 483                          (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
 484                          (u32) cpu->acpi_perf_data.states[i].core_frequency,
 485                          (u32) cpu->acpi_perf_data.states[i].power,
 486                          (u32) cpu->acpi_perf_data.states[i].control);
 487         }
 488
 489         /*
 490          * The _PSS table doesn't contain whole turbo frequency range.
 491          * This just contains +1 MHZ above the max non turbo frequency,
 492          * with control value corresponding to max turbo ratio. But
 493          * when cpufreq set policy is called, it will call with this
 494          * max frequency, which will cause a reduced performance as
 495          * this driver uses real max turbo frequency as the max
 496          * frequency. So correct this frequency in _PSS table to
 497          * correct max turbo frequency based on the turbo state.
 498          * Also need to convert to MHz as _PSS freq is in MHz.
 499          */
 500         if (!global.turbo_disabled)
 501                 cpu->acpi_perf_data.states[0].core_frequency =
 502                                         policy->cpuinfo.max_freq / 1000;
 503         cpu->valid_pss_table = true;
 504         pr_debug("_PPC limits will be enforced\n");
 505
 506         return;
 507
 508  err:
 509         cpu->valid_pss_table = false;
 510         acpi_processor_unregister_performance(policy->cpu);
 511 }
 512
 513 static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 514 {
 515         struct cpudata *cpu;
 516
 517         cpu = all_cpu_data[policy->cpu];
 518         if (!cpu->valid_pss_table)
 519                 return;
 520
 521         acpi_processor_unregister_performance(policy->cpu);
 522 }
 523 #else
 524 static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
 525 {
 526 }
 527
 528 static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
 529 {
 530 }
 531 #endif
 532
 533 static inline void pid_reset(struct _pid *pid, int setpoint, int busy,
 534                              int deadband, int integral) {
 535         pid->setpoint = int_tofp(setpoint);
 536         pid->deadband  = int_tofp(deadband);
 537         pid->integral  = int_tofp(integral);
 538         pid->last_err  = int_tofp(setpoint) - int_tofp(busy);
 539 }
 540
 541 static inline void pid_p_gain_set(struct _pid *pid, int percent)
 542 {
 543         pid->p_gain = div_fp(percent, 100);
 544 }
 545
 546 static inline void pid_i_gain_set(struct _pid *pid, int percent)
 547 {
 548         pid->i_gain = div_fp(percent, 100);
 549 }
 550
 551 static inline void pid_d_gain_set(struct _pid *pid, int percent)
 552 {
 553         pid->d_gain = div_fp(percent, 100);
 554 }
 555
 556 static signed int pid_calc(struct _pid *pid, int32_t busy)
 557 {
 558         signed int result;
 559         int32_t pterm, dterm, fp_error;
 560         int32_t integral_limit;
 561
 562         fp_error = pid->setpoint - busy;
 563
 564         if (abs(fp_error) <= pid->deadband)
 565                 return 0;
 566
 567         pterm = mul_fp(pid->p_gain, fp_error);
 568
 569         pid->integral += fp_error;
 570
 571         /*
 572          * We limit the integral here so that it will never
 573          * get higher than 30.  This prevents it from becoming
 574          * too large an input over long periods of time and allows
 575          * it to get factored out sooner.
 576          *
 577          * The value of 30 was chosen through experimentation.
 578          */
 579         integral_limit = int_tofp(30);
 580         if (pid->integral > integral_limit)
 581                 pid->integral = integral_limit;
 582         if (pid->integral < -integral_limit)
 583                 pid->integral = -integral_limit;
 584
 585         dterm = mul_fp(pid->d_gain, fp_error - pid->last_err);
 586         pid->last_err = fp_error;
 587
 588         result = pterm + mul_fp(pid->integral, pid->i_gain) + dterm;
 589         result = result + (1 << (FRAC_BITS-1));
 590         return (signed int)fp_toint(result);
 591 }
 592
 593 static inline void intel_pstate_busy_pid_reset(struct cpudata *cpu)
 594 {
 595         pid_p_gain_set(&cpu->pid, pid_params.p_gain_pct);
 596         pid_d_gain_set(&cpu->pid, pid_params.d_gain_pct);
 597         pid_i_gain_set(&cpu->pid, pid_params.i_gain_pct);
 598
 599         pid_reset(&cpu->pid, pid_params.setpoint, 100, pid_params.deadband, 0);
 600 }
 601
 602 static inline void intel_pstate_reset_all_pid(void)
 603 {
 604         unsigned int cpu;
 605
 606         for_each_online_cpu(cpu) {
 607                 if (all_cpu_data[cpu])
 608                         intel_pstate_busy_pid_reset(all_cpu_data[cpu]);
 609         }
 610 }
 611
 612 static inline void update_turbo_state(void)
 613 {
 614         u64 misc_en;
 615         struct cpudata *cpu;
 616
 617         cpu = all_cpu_data[0];
 618         rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
 619         global.turbo_disabled =
 620                 (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
 621                  cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
 622 }
 623
 624 static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
 625 {
 626         u64 epb;
 627         int ret;
 628
 629         if (!static_cpu_has(X86_FEATURE_EPB))
 630                 return -ENXIO;
 631
 632         ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
 633         if (ret)
 634                 return (s16)ret;
 635
 636         return (s16)(epb & 0x0f);
 637 }
 638
 639 static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)
 640 {
 641         s16 epp;
 642
 643         if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 644                 /*
 645                  * When hwp_req_data is 0, means that caller didn't read
 646                  * MSR_HWP_REQUEST, so need to read and get EPP.
 647                  */
 648                 if (!hwp_req_data) {
 649                         epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST,
 650                                             &hwp_req_data);
 651                         if (epp)
 652                                 return epp;
 653                 }
 654                 epp = (hwp_req_data >> 24) & 0xff;
 655         } else {
 656                 /* When there is no EPP present, HWP uses EPB settings */
 657                 epp = intel_pstate_get_epb(cpu_data);
 658         }
 659
 660         return epp;
 661 }
 662
 663 static int intel_pstate_set_epb(int cpu, s16 pref)
 664 {
 665         u64 epb;
 666         int ret;
 667
 668         if (!static_cpu_has(X86_FEATURE_EPB))
 669                 return -ENXIO;
 670
 671         ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
 672         if (ret)
 673                 return ret;
 674
 675         epb = (epb & ~0x0f) | pref;
 676         wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb);
 677
 678         return 0;
 679 }
 680
 681 /*
 682  * EPP/EPB display strings corresponding to EPP index in the
 683  * energy_perf_strings[]
 684  *      index           String
 685  *-------------------------------------
 686  *      0               default
 687  *      1               performance
 688  *      2               balance_performance
 689  *      3               balance_power
 690  *      4               power
 691  */
 692 static const char * const energy_perf_strings[] = {
 693         "default",
 694         "performance",
 695         "balance_performance",
 696         "balance_power",
 697         "power",
 698         NULL
 699 };
 700
 701 static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data)
 702 {
 703         s16 epp;
 704         int index = -EINVAL;
 705
 706         epp = intel_pstate_get_epp(cpu_data, 0);
 707         if (epp < 0)
 708                 return epp;
 709
 710         if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 711                 /*
 712                  * Range:
 713                  *      0x00-0x3F       :       Performance
 714                  *      0x40-0x7F       :       Balance performance
 715                  *      0x80-0xBF       :       Balance power
 716                  *      0xC0-0xFF       :       Power
 717                  * The EPP is a 8 bit value, but our ranges restrict the
 718                  * value which can be set. Here only using top two bits
 719                  * effectively.
 720                  */
 721                 index = (epp >> 6) + 1;
 722         } else if (static_cpu_has(X86_FEATURE_EPB)) {
 723                 /*
 724                  * Range:
 725                  *      0x00-0x03       :       Performance
 726                  *      0x04-0x07       :       Balance performance
 727                  *      0x08-0x0B       :       Balance power
 728                  *      0x0C-0x0F       :       Power
 729                  * The EPB is a 4 bit value, but our ranges restrict the
 730                  * value which can be set. Here only using top two bits
 731                  * effectively.
 732                  */
 733                 index = (epp >> 2) + 1;
 734         }
 735
 736         return index;
 737 }
 738
 739 static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
 740                                               int pref_index)
 741 {
 742         int epp = -EINVAL;
 743         int ret;
 744
 745         if (!pref_index)
 746                 epp = cpu_data->epp_default;
 747
 748         mutex_lock(&intel_pstate_limits_lock);
 749
 750         if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 751                 u64 value;
 752
 753                 ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, &value);
 754                 if (ret)
 755                         goto return_pref;
 756
 757                 value &= ~GENMASK_ULL(31, 24);
 758
 759                 /*
 760                  * If epp is not default, convert from index into
 761                  * energy_perf_strings to epp value, by shifting 6
 762                  * bits left to use only top two bits in epp.
 763                  * The resultant epp need to shifted by 24 bits to
 764                  * epp position in MSR_HWP_REQUEST.
 765                  */
 766                 if (epp == -EINVAL)
 767                         epp = (pref_index - 1) << 6;
 768
 769                 value |= (u64)epp << 24;
 770                 ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value);
 771         } else {
 772                 if (epp == -EINVAL)
 773                         epp = (pref_index - 1) << 2;
 774                 ret = intel_pstate_set_epb(cpu_data->cpu, epp);
 775         }
 776 return_pref:
 777         mutex_unlock(&intel_pstate_limits_lock);
 778
 779         return ret;
 780 }
 781
 782 static ssize_t show_energy_performance_available_preferences(
 783                                 struct cpufreq_policy *policy, char *buf)
 784 {
 785         int i = 0;
 786         int ret = 0;
 787
 788         while (energy_perf_strings[i] != NULL)
 789                 ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]);
 790
 791         ret += sprintf(&buf[ret], "\n");
 792
 793         return ret;
 794 }
 795
 796 cpufreq_freq_attr_ro(energy_performance_available_preferences);
 797
 798 static ssize_t store_energy_performance_preference(
 799                 struct cpufreq_policy *policy, const char *buf, size_t count)
 800 {
 801         struct cpudata *cpu_data = all_cpu_data[policy->cpu];
 802         char str_preference[21];
 803         int ret, i = 0;
 804
 805         ret = sscanf(buf, "%20s", str_preference);
 806         if (ret != 1)
 807                 return -EINVAL;
 808
 809         while (energy_perf_strings[i] != NULL) {
 810                 if (!strcmp(str_preference, energy_perf_strings[i])) {
 811                         intel_pstate_set_energy_pref_index(cpu_data, i);
 812                         return count;
 813                 }
 814                 ++i;
 815         }
 816
 817         return -EINVAL;
 818 }
 819
 820 static ssize_t show_energy_performance_preference(
 821                                 struct cpufreq_policy *policy, char *buf)
 822 {
 823         struct cpudata *cpu_data = all_cpu_data[policy->cpu];
 824         int preference;
 825
 826         preference = intel_pstate_get_energy_pref_index(cpu_data);
 827         if (preference < 0)
 828                 return preference;
 829
 830         return  sprintf(buf, "%s\n", energy_perf_strings[preference]);
 831 }
 832
 833 cpufreq_freq_attr_rw(energy_performance_preference);
 834
 835 static struct freq_attr *hwp_cpufreq_attrs[] = {
 836         &energy_performance_preference,
 837         &energy_performance_available_preferences,
 838         NULL,
 839 };
 840
 841 static void intel_pstate_hwp_set(struct cpufreq_policy *policy)
 842 {
 843         int min, hw_min, max, hw_max, cpu;
 844         struct perf_limits *perf_limits = &global;
 845         u64 value, cap;
 846
 847         for_each_cpu(cpu, policy->cpus) {
 848                 struct cpudata *cpu_data = all_cpu_data[cpu];
 849                 s16 epp;
 850
 851                 if (per_cpu_limits)
 852                         perf_limits = all_cpu_data[cpu]->perf_limits;
 853
 854                 rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap);
 855                 hw_min = HWP_LOWEST_PERF(cap);
 856                 if (global.no_turbo)
 857                         hw_max = HWP_GUARANTEED_PERF(cap);
 858                 else
 859                         hw_max = HWP_HIGHEST_PERF(cap);
 860
 861                 max = fp_ext_toint(hw_max * perf_limits->max_perf);
 862                 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
 863                         min = max;
 864                 else
 865                         min = fp_ext_toint(hw_max * perf_limits->min_perf);
 866
 867                 rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
 868
 869                 value &= ~HWP_MIN_PERF(~0L);
 870                 value |= HWP_MIN_PERF(min);
 871
 872                 value &= ~HWP_MAX_PERF(~0L);
 873                 value |= HWP_MAX_PERF(max);
 874
 875                 if (cpu_data->epp_policy == cpu_data->policy)
 876                         goto skip_epp;
 877
 878                 cpu_data->epp_policy = cpu_data->policy;
 879
 880                 if (cpu_data->epp_saved >= 0) {
 881                         epp = cpu_data->epp_saved;
 882                         cpu_data->epp_saved = -EINVAL;
 883                         goto update_epp;
 884                 }
 885
 886                 if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
 887                         epp = intel_pstate_get_epp(cpu_data, value);
 888                         cpu_data->epp_powersave = epp;
 889                         /* If EPP read was failed, then don't try to write */
 890                         if (epp < 0)
 891                                 goto skip_epp;
 892
 893
 894                         epp = 0;
 895                 } else {
 896                         /* skip setting EPP, when saved value is invalid */
 897                         if (cpu_data->epp_powersave < 0)
 898                                 goto skip_epp;
 899
 900                         /*
 901                          * No need to restore EPP when it is not zero. This
 902                          * means:
 903                          *  - Policy is not changed
 904                          *  - user has manually changed
 905                          *  - Error reading EPB
 906                          */
 907                         epp = intel_pstate_get_epp(cpu_data, value);
 908                         if (epp)
 909                                 goto skip_epp;
 910
 911                         epp = cpu_data->epp_powersave;
 912                 }
 913 update_epp:
 914                 if (static_cpu_has(X86_FEATURE_HWP_EPP)) {
 915                         value &= ~GENMASK_ULL(31, 24);
 916                         value |= (u64)epp << 24;
 917                 } else {
 918                         intel_pstate_set_epb(cpu, epp);
 919                 }
 920 skip_epp:
 921                 wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
 922         }
 923 }
 924
 925 static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy)
 926 {
 927         if (hwp_active)
 928                 intel_pstate_hwp_set(policy);
 929
 930         return 0;
 931 }
 932
 933 static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy)
 934 {
 935         struct cpudata *cpu_data = all_cpu_data[policy->cpu];
 936
 937         if (!hwp_active)
 938                 return 0;
 939
 940         cpu_data->epp_saved = intel_pstate_get_epp(cpu_data, 0);
 941
 942         return 0;
 943 }
 944
 945 static int intel_pstate_resume(struct cpufreq_policy *policy)
 946 {
 947         int ret;
 948
 949         if (!hwp_active)
 950                 return 0;
 951
 952         mutex_lock(&intel_pstate_limits_lock);
 953
 954         all_cpu_data[policy->cpu]->epp_policy = 0;
 955
 956         ret = intel_pstate_hwp_set_policy(policy);
 957
 958         mutex_unlock(&intel_pstate_limits_lock);
 959
 960         return ret;
 961 }
 962
 963 static void intel_pstate_update_policies(void)
 964 {
 965         int cpu;
 966
 967         for_each_possible_cpu(cpu)
 968                 cpufreq_update_policy(cpu);
 969 }
 970
 971 /************************** debugfs begin ************************/
 972 static int pid_param_set(void *data, u64 val)
 973 {
 974         *(u32 *)data = val;
 975         pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
 976         intel_pstate_reset_all_pid();
 977         return 0;
 978 }
 979
 980 static int pid_param_get(void *data, u64 *val)
 981 {
 982         *val = *(u32 *)data;
 983         return 0;
 984 }
 985 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param, pid_param_get, pid_param_set, "%llu\n");
 986
 987 static struct dentry *debugfs_parent;
 988
 989 struct pid_param {
 990         char *name;
 991         void *value;
 992         struct dentry *dentry;
 993 };
 994
 995 static struct pid_param pid_files[] = {
 996         {"sample_rate_ms", &pid_params.sample_rate_ms, },
 997         {"d_gain_pct", &pid_params.d_gain_pct, },
 998         {"i_gain_pct", &pid_params.i_gain_pct, },
 999         {"deadband", &pid_params.deadband, },
1000         {"setpoint", &pid_params.setpoint, },
1001         {"p_gain_pct", &pid_params.p_gain_pct, },
1002         {NULL, NULL, }
1003 };
1004
1005 static void intel_pstate_debug_expose_params(void)
1006 {
1007         int i;
1008
1009         debugfs_parent = debugfs_create_dir("pstate_snb", NULL);
1010         if (IS_ERR_OR_NULL(debugfs_parent))
1011                 return;
1012
1013         for (i = 0; pid_files[i].name; i++) {
1014                 struct dentry *dentry;
1015
1016                 dentry = debugfs_create_file(pid_files[i].name, 0660,
1017                                              debugfs_parent, pid_files[i].value,
1018                                              &fops_pid_param);
1019                 if (!IS_ERR(dentry))
1020                         pid_files[i].dentry = dentry;
1021         }
1022 }
1023
1024 static void intel_pstate_debug_hide_params(void)
1025 {
1026         int i;
1027
1028         if (IS_ERR_OR_NULL(debugfs_parent))
1029                 return;
1030
1031         for (i = 0; pid_files[i].name; i++) {
1032                 debugfs_remove(pid_files[i].dentry);
1033                 pid_files[i].dentry = NULL;
1034         }
1035
1036         debugfs_remove(debugfs_parent);
1037         debugfs_parent = NULL;
1038 }
1039
1040 /************************** debugfs end ************************/
1041
1042 /************************** sysfs begin ************************/
1043 #define show_one(file_name, object)                                     \
1044         static ssize_t show_##file_name                                 \
1045         (struct kobject *kobj, struct attribute *attr, char *buf)       \
1046         {                                                               \
1047                 return sprintf(buf, "%u\n", global.object);             \
1048         }
1049
1050 static ssize_t intel_pstate_show_status(char *buf);
1051 static int intel_pstate_update_status(const char *buf, size_t size);
1052
1053 static ssize_t show_status(struct kobject *kobj,
1054                            struct attribute *attr, char *buf)
1055 {
1056         ssize_t ret;
1057
1058         mutex_lock(&intel_pstate_driver_lock);
1059         ret = intel_pstate_show_status(buf);
1060         mutex_unlock(&intel_pstate_driver_lock);
1061
1062         return ret;
1063 }
1064
1065 static ssize_t store_status(struct kobject *a, struct attribute *b,
1066                             const char *buf, size_t count)
1067 {
1068         char *p = memchr(buf, '\n', count);
1069         int ret;
1070
1071         mutex_lock(&intel_pstate_driver_lock);
1072         ret = intel_pstate_update_status(buf, p ? p - buf : count);
1073         mutex_unlock(&intel_pstate_driver_lock);
1074
1075         return ret < 0 ? ret : count;
1076 }
1077
1078 static ssize_t show_turbo_pct(struct kobject *kobj,
1079                                 struct attribute *attr, char *buf)
1080 {
1081         struct cpudata *cpu;
1082         int total, no_turbo, turbo_pct;
1083         uint32_t turbo_fp;
1084
1085         mutex_lock(&intel_pstate_driver_lock);
1086
1087         if (!driver_registered) {
1088                 mutex_unlock(&intel_pstate_driver_lock);
1089                 return -EAGAIN;
1090         }
1091
1092         cpu = all_cpu_data[0];
1093
1094         total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
1095         no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
1096         turbo_fp = div_fp(no_turbo, total);
1097         turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
1098
1099         mutex_unlock(&intel_pstate_driver_lock);
1100
1101         return sprintf(buf, "%u\n", turbo_pct);
1102 }
1103
1104 static ssize_t show_num_pstates(struct kobject *kobj,
1105                                 struct attribute *attr, char *buf)
1106 {
1107         struct cpudata *cpu;
1108         int total;
1109
1110         mutex_lock(&intel_pstate_driver_lock);
1111
1112         if (!driver_registered) {
1113                 mutex_unlock(&intel_pstate_driver_lock);
1114                 return -EAGAIN;
1115         }
1116
1117         cpu = all_cpu_data[0];
1118         total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
1119
1120         mutex_unlock(&intel_pstate_driver_lock);
1121
1122         return sprintf(buf, "%u\n", total);
1123 }
1124
1125 static ssize_t show_no_turbo(struct kobject *kobj,
1126                              struct attribute *attr, char *buf)
1127 {
1128         ssize_t ret;
1129
1130         mutex_lock(&intel_pstate_driver_lock);
1131
1132         if (!driver_registered) {
1133                 mutex_unlock(&intel_pstate_driver_lock);
1134                 return -EAGAIN;
1135         }
1136
1137         update_turbo_state();
1138         if (global.turbo_disabled)
1139                 ret = sprintf(buf, "%u\n", global.turbo_disabled);
1140         else
1141                 ret = sprintf(buf, "%u\n", global.no_turbo);
1142
1143         mutex_unlock(&intel_pstate_driver_lock);
1144
1145         return ret;
1146 }
1147
1148 static ssize_t store_no_turbo(struct kobject *a, struct attribute *b,
1149                               const char *buf, size_t count)
1150 {
1151         unsigned int input;
1152         int ret;
1153
1154         ret = sscanf(buf, "%u", &input);
1155         if (ret != 1)
1156                 return -EINVAL;
1157
1158         mutex_lock(&intel_pstate_driver_lock);
1159
1160         if (!driver_registered) {
1161                 mutex_unlock(&intel_pstate_driver_lock);
1162                 return -EAGAIN;
1163         }
1164
1165         mutex_lock(&intel_pstate_limits_lock);
1166
1167         update_turbo_state();
1168         if (global.turbo_disabled) {
1169                 pr_warn("Turbo disabled by BIOS or unavailable on processor\n");
1170                 mutex_unlock(&intel_pstate_limits_lock);
1171                 mutex_unlock(&intel_pstate_driver_lock);
1172                 return -EPERM;
1173         }
1174
1175         global.no_turbo = clamp_t(int, input, 0, 1);
1176
1177         mutex_unlock(&intel_pstate_limits_lock);
1178
1179         intel_pstate_update_policies();
1180
1181         mutex_unlock(&intel_pstate_driver_lock);
1182
1183         return count;
1184 }
1185
1186 static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b,
1187                                   const char *buf, size_t count)
1188 {
1189         unsigned int input;
1190         int ret;
1191
1192         ret = sscanf(buf, "%u", &input);
1193         if (ret != 1)
1194                 return -EINVAL;
1195
1196         mutex_lock(&intel_pstate_driver_lock);
1197
1198         if (!driver_registered) {
1199                 mutex_unlock(&intel_pstate_driver_lock);
1200                 return -EAGAIN;
1201         }
1202
1203         mutex_lock(&intel_pstate_limits_lock);
1204
1205         global.max_sysfs_pct = clamp_t(int, input, 0 , 100);
1206         global.max_perf_pct = min(global.max_policy_pct, global.max_sysfs_pct);
1207         global.max_perf_pct = max(global.min_policy_pct, global.max_perf_pct);
1208         global.max_perf_pct = max(global.min_perf_pct, global.max_perf_pct);
1209         global.max_perf = percent_ext_fp(global.max_perf_pct);
1210
1211         mutex_unlock(&intel_pstate_limits_lock);
1212
1213         intel_pstate_update_policies();
1214
1215         mutex_unlock(&intel_pstate_driver_lock);
1216
1217         return count;
1218 }
1219
1220 static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b,
1221                                   const char *buf, size_t count)
1222 {
1223         unsigned int input;
1224         int ret;
1225
1226         ret = sscanf(buf, "%u", &input);
1227         if (ret != 1)
1228                 return -EINVAL;
1229
1230         mutex_lock(&intel_pstate_driver_lock);
1231
1232         if (!driver_registered) {
1233                 mutex_unlock(&intel_pstate_driver_lock);
1234                 return -EAGAIN;
1235         }
1236
1237         mutex_lock(&intel_pstate_limits_lock);
1238
1239         global.min_sysfs_pct = clamp_t(int, input, 0 , 100);
1240         global.min_perf_pct = max(global.min_policy_pct, global.min_sysfs_pct);
1241         global.min_perf_pct = min(global.max_policy_pct, global.min_perf_pct);
1242         global.min_perf_pct = min(global.max_perf_pct, global.min_perf_pct);
1243         global.min_perf = percent_ext_fp(global.min_perf_pct);
1244
1245         mutex_unlock(&intel_pstate_limits_lock);
1246
1247         intel_pstate_update_policies();
1248
1249         mutex_unlock(&intel_pstate_driver_lock);
1250
1251         return count;
1252 }
1253
1254 show_one(max_perf_pct, max_perf_pct);
1255 show_one(min_perf_pct, min_perf_pct);
1256
1257 define_one_global_rw(status);
1258 define_one_global_rw(no_turbo);
1259 define_one_global_rw(max_perf_pct);
1260 define_one_global_rw(min_perf_pct);
1261 define_one_global_ro(turbo_pct);
1262 define_one_global_ro(num_pstates);
1263
1264 static struct attribute *intel_pstate_attributes[] = {
1265         &status.attr,
1266         &no_turbo.attr,
1267         &turbo_pct.attr,
1268         &num_pstates.attr,
1269         NULL
1270 };
1271
1272 static struct attribute_group intel_pstate_attr_group = {
1273         .attrs = intel_pstate_attributes,
1274 };
1275
1276 static void __init intel_pstate_sysfs_expose_params(void)
1277 {
1278         struct kobject *intel_pstate_kobject;
1279         int rc;
1280
1281         intel_pstate_kobject = kobject_create_and_add("intel_pstate",
1282                                                 &cpu_subsys.dev_root->kobj);
1283         if (WARN_ON(!intel_pstate_kobject))
1284                 return;
1285
1286         rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
1287         if (WARN_ON(rc))
1288                 return;
1289
1290         /*
1291          * If per cpu limits are enforced there are no global limits, so
1292          * return without creating max/min_perf_pct attributes
1293          */
1294         if (per_cpu_limits)
1295                 return;
1296
1297         rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
1298         WARN_ON(rc);
1299
1300         rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
1301         WARN_ON(rc);
1302
1303 }
1304 /************************** sysfs end ************************/
1305
1306 static void intel_pstate_hwp_enable(struct cpudata *cpudata)
1307 {
1308         /* First disable HWP notification interrupt as we don't process them */
1309         if (static_cpu_has(X86_FEATURE_HWP_NOTIFY))
1310                 wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
1311
1312         wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
1313         cpudata->epp_policy = 0;
1314         if (cpudata->epp_default == -EINVAL)
1315                 cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
1316 }
1317
1318 #define MSR_IA32_POWER_CTL_BIT_EE       19
1319
1320 /* Disable energy efficiency optimization */
1321 static void intel_pstate_disable_ee(int cpu)
1322 {
1323         u64 power_ctl;
1324         int ret;
1325
1326         ret = rdmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, &power_ctl);
1327         if (ret)
1328                 return;
1329
1330         if (!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE))) {
1331                 pr_info("Disabling energy efficiency optimization\n");
1332                 power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
1333                 wrmsrl_on_cpu(cpu, MSR_IA32_POWER_CTL, power_ctl);
1334         }
1335 }
1336
1337 static int atom_get_min_pstate(void)
1338 {
1339         u64 value;
1340
1341         rdmsrl(MSR_ATOM_CORE_RATIOS, value);
1342         return (value >> 8) & 0x7F;
1343 }
1344
1345 static int atom_get_max_pstate(void)
1346 {
1347         u64 value;
1348
1349         rdmsrl(MSR_ATOM_CORE_RATIOS, value);
1350         return (value >> 16) & 0x7F;
1351 }
1352
1353 static int atom_get_turbo_pstate(void)
1354 {
1355         u64 value;
1356
1357         rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value);
1358         return value & 0x7F;
1359 }
1360
1361 static u64 atom_get_val(struct cpudata *cpudata, int pstate)
1362 {
1363         u64 val;
1364         int32_t vid_fp;
1365         u32 vid;
1366
1367         val = (u64)pstate << 8;
1368         if (global.no_turbo && !global.turbo_disabled)
1369                 val |= (u64)1 << 32;
1370
1371         vid_fp = cpudata->vid.min + mul_fp(
1372                 int_tofp(pstate - cpudata->pstate.min_pstate),
1373                 cpudata->vid.ratio);
1374
1375         vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
1376         vid = ceiling_fp(vid_fp);
1377
1378         if (pstate > cpudata->pstate.max_pstate)
1379                 vid = cpudata->vid.turbo;
1380
1381         return val | vid;
1382 }
1383
1384 static int silvermont_get_scaling(void)
1385 {
1386         u64 value;
1387         int i;
1388         /* Defined in Table 35-6 from SDM (Sept 2015) */
1389         static int silvermont_freq_table[] = {
1390                 83300, 100000, 133300, 116700, 80000};
1391
1392         rdmsrl(MSR_FSB_FREQ, value);
1393         i = value & 0x7;
1394         WARN_ON(i > 4);
1395
1396         return silvermont_freq_table[i];
1397 }
1398
1399 static int airmont_get_scaling(void)
1400 {
1401         u64 value;
1402         int i;
1403         /* Defined in Table 35-10 from SDM (Sept 2015) */
1404         static int airmont_freq_table[] = {
1405                 83300, 100000, 133300, 116700, 80000,
1406                 93300, 90000, 88900, 87500};
1407
1408         rdmsrl(MSR_FSB_FREQ, value);
1409         i = value & 0xF;
1410         WARN_ON(i > 8);
1411
1412         return airmont_freq_table[i];
1413 }
1414
1415 static void atom_get_vid(struct cpudata *cpudata)
1416 {
1417         u64 value;
1418
1419         rdmsrl(MSR_ATOM_CORE_VIDS, value);
1420         cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
1421         cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
1422         cpudata->vid.ratio = div_fp(
1423                 cpudata->vid.max - cpudata->vid.min,
1424                 int_tofp(cpudata->pstate.max_pstate -
1425                         cpudata->pstate.min_pstate));
1426
1427         rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value);
1428         cpudata->vid.turbo = value & 0x7f;
1429 }
1430
1431 static int core_get_min_pstate(void)
1432 {
1433         u64 value;
1434
1435         rdmsrl(MSR_PLATFORM_INFO, value);
1436         return (value >> 40) & 0xFF;
1437 }
1438
1439 static int core_get_max_pstate_physical(void)
1440 {
1441         u64 value;
1442
1443         rdmsrl(MSR_PLATFORM_INFO, value);
1444         return (value >> 8) & 0xFF;
1445 }
1446
1447 static int core_get_tdp_ratio(u64 plat_info)
1448 {
1449         /* Check how many TDP levels present */
1450         if (plat_info & 0x600000000) {
1451                 u64 tdp_ctrl;
1452                 u64 tdp_ratio;
1453                 int tdp_msr;
1454                 int err;
1455
1456                 /* Get the TDP level (0, 1, 2) to get ratios */
1457                 err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
1458                 if (err)
1459                         return err;
1460
1461                 /* TDP MSR are continuous starting at 0x648 */
1462                 tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03);
1463                 err = rdmsrl_safe(tdp_msr, &tdp_ratio);
1464                 if (err)
1465                         return err;
1466
1467                 /* For level 1 and 2, bits[23:16] contain the ratio */
1468                 if (tdp_ctrl & 0x03)
1469                         tdp_ratio >>= 16;
1470
1471                 tdp_ratio &= 0xff; /* ratios are only 8 bits long */
1472                 pr_debug("tdp_ratio %x\n", (int)tdp_ratio);
1473
1474                 return (int)tdp_ratio;
1475         }
1476
1477         return -ENXIO;
1478 }
1479
1480 static int core_get_max_pstate(void)
1481 {
1482         u64 tar;
1483         u64 plat_info;
1484         int max_pstate;
1485         int tdp_ratio;
1486         int err;
1487
1488         rdmsrl(MSR_PLATFORM_INFO, plat_info);
1489         max_pstate = (plat_info >> 8) & 0xFF;
1490
1491         tdp_ratio = core_get_tdp_ratio(plat_info);
1492         if (tdp_ratio <= 0)
1493                 return max_pstate;
1494
1495         if (hwp_active) {
1496                 /* Turbo activation ratio is not used on HWP platforms */
1497                 return tdp_ratio;
1498         }
1499
1500         err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
1501         if (!err) {
1502                 int tar_levels;
1503
1504                 /* Do some sanity checking for safety */
1505                 tar_levels = tar & 0xff;
1506                 if (tdp_ratio - 1 == tar_levels) {
1507                         max_pstate = tar_levels;
1508                         pr_debug("max_pstate=TAC %x\n", max_pstate);
1509                 }
1510         }
1511
1512         return max_pstate;
1513 }
1514
1515 static int core_get_turbo_pstate(void)
1516 {
1517         u64 value;
1518         int nont, ret;
1519
1520         rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
1521         nont = core_get_max_pstate();
1522         ret = (value) & 255;
1523         if (ret <= nont)
1524                 ret = nont;
1525         return ret;
1526 }
1527
1528 static inline int core_get_scaling(void)
1529 {
1530         return 100000;
1531 }
1532
1533 static u64 core_get_val(struct cpudata *cpudata, int pstate)
1534 {
1535         u64 val;
1536
1537         val = (u64)pstate << 8;
1538         if (global.no_turbo && !global.turbo_disabled)
1539                 val |= (u64)1 << 32;
1540
1541         return val;
1542 }
1543
1544 static int knl_get_turbo_pstate(void)
1545 {
1546         u64 value;
1547         int nont, ret;
1548
1549         rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
1550         nont = core_get_max_pstate();
1551         ret = (((value) >> 8) & 0xFF);
1552         if (ret <= nont)
1553                 ret = nont;
1554         return ret;
1555 }
1556
1557 static struct cpu_defaults core_params = {
1558         .pid_policy = {
1559                 .sample_rate_ms = 10,
1560                 .deadband = 0,
1561                 .setpoint = 97,
1562                 .p_gain_pct = 20,
1563                 .d_gain_pct = 0,
1564                 .i_gain_pct = 0,
1565         },
1566         .funcs = {
1567                 .get_max = core_get_max_pstate,
1568                 .get_max_physical = core_get_max_pstate_physical,
1569                 .get_min = core_get_min_pstate,
1570                 .get_turbo = core_get_turbo_pstate,
1571                 .get_scaling = core_get_scaling,
1572                 .get_val = core_get_val,
1573                 .get_target_pstate = get_target_pstate_use_performance,
1574         },
1575 };
1576
1577 static const struct cpu_defaults silvermont_params = {
1578         .pid_policy = {
1579                 .sample_rate_ms = 10,
1580                 .deadband = 0,
1581                 .setpoint = 60,
1582                 .p_gain_pct = 14,
1583                 .d_gain_pct = 0,
1584                 .i_gain_pct = 4,
1585         },
1586         .funcs = {
1587                 .get_max = atom_get_max_pstate,
1588                 .get_max_physical = atom_get_max_pstate,
1589                 .get_min = atom_get_min_pstate,
1590                 .get_turbo = atom_get_turbo_pstate,
1591                 .get_val = atom_get_val,
1592                 .get_scaling = silvermont_get_scaling,
1593                 .get_vid = atom_get_vid,
1594                 .get_target_pstate = get_target_pstate_use_cpu_load,
1595         },
1596 };
1597
1598 static const struct cpu_defaults airmont_params = {
1599         .pid_policy = {
1600                 .sample_rate_ms = 10,
1601                 .deadband = 0,
1602                 .setpoint = 60,
1603                 .p_gain_pct = 14,
1604                 .d_gain_pct = 0,
1605                 .i_gain_pct = 4,
1606         },
1607         .funcs = {
1608                 .get_max = atom_get_max_pstate,
1609                 .get_max_physical = atom_get_max_pstate,
1610                 .get_min = atom_get_min_pstate,
1611                 .get_turbo = atom_get_turbo_pstate,
1612                 .get_val = atom_get_val,
1613                 .get_scaling = airmont_get_scaling,
1614                 .get_vid = atom_get_vid,
1615                 .get_target_pstate = get_target_pstate_use_cpu_load,
1616         },
1617 };
1618
1619 static const struct cpu_defaults knl_params = {
1620         .pid_policy = {
1621                 .sample_rate_ms = 10,
1622                 .deadband = 0,
1623                 .setpoint = 97,
1624                 .p_gain_pct = 20,
1625                 .d_gain_pct = 0,
1626                 .i_gain_pct = 0,
1627         },
1628         .funcs = {
1629                 .get_max = core_get_max_pstate,
1630                 .get_max_physical = core_get_max_pstate_physical,
1631                 .get_min = core_get_min_pstate,
1632                 .get_turbo = knl_get_turbo_pstate,
1633                 .get_scaling = core_get_scaling,
1634                 .get_val = core_get_val,
1635                 .get_target_pstate = get_target_pstate_use_performance,
1636         },
1637 };
1638
1639 static const struct cpu_defaults bxt_params = {
1640         .pid_policy = {
1641                 .sample_rate_ms = 10,
1642                 .deadband = 0,
1643                 .setpoint = 60,
1644                 .p_gain_pct = 14,
1645                 .d_gain_pct = 0,
1646                 .i_gain_pct = 4,
1647         },
1648         .funcs = {
1649                 .get_max = core_get_max_pstate,
1650                 .get_max_physical = core_get_max_pstate_physical,
1651                 .get_min = core_get_min_pstate,
1652                 .get_turbo = core_get_turbo_pstate,
1653                 .get_scaling = core_get_scaling,
1654                 .get_val = core_get_val,
1655                 .get_target_pstate = get_target_pstate_use_cpu_load,
1656         },
1657 };
1658
1659 static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max)
1660 {
1661         int max_perf = cpu->pstate.turbo_pstate;
1662         int max_perf_adj;
1663         int min_perf;
1664         struct perf_limits *perf_limits = &global;
1665
1666         if (global.no_turbo || global.turbo_disabled)
1667                 max_perf = cpu->pstate.max_pstate;
1668
1669         if (per_cpu_limits)
1670                 perf_limits = cpu->perf_limits;
1671
1672         /*
1673          * performance can be limited by user through sysfs, by cpufreq
1674          * policy, or by cpu specific default values determined through
1675          * experimentation.
1676          */
1677         max_perf_adj = fp_ext_toint(max_perf * perf_limits->max_perf);
1678         *max = clamp_t(int, max_perf_adj,
1679                         cpu->pstate.min_pstate, cpu->pstate.turbo_pstate);
1680
1681         min_perf = fp_ext_toint(max_perf * perf_limits->min_perf);
1682         *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf);
1683 }
1684
1685 static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
1686 {
1687         trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
1688         cpu->pstate.current_pstate = pstate;
1689         /*
1690          * Generally, there is no guarantee that this code will always run on
1691          * the CPU being updated, so force the register update to run on the
1692          * right CPU.
1693          */
1694         wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
1695                       pstate_funcs.get_val(cpu, pstate));
1696 }
1697
1698 static void intel_pstate_set_min_pstate(struct cpudata *cpu)
1699 {
1700         intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
1701 }
1702
1703 static void intel_pstate_max_within_limits(struct cpudata *cpu)
1704 {
1705         int min_pstate, max_pstate;
1706
1707         update_turbo_state();
1708         intel_pstate_get_min_max(cpu, &min_pstate, &max_pstate);
1709         intel_pstate_set_pstate(cpu, max_pstate);
1710 }
1711
1712 static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
1713 {
1714         cpu->pstate.min_pstate = pstate_funcs.get_min();
1715         cpu->pstate.max_pstate = pstate_funcs.get_max();
1716         cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
1717         cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
1718         cpu->pstate.scaling = pstate_funcs.get_scaling();
1719         cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling;
1720         cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
1721
1722         if (pstate_funcs.get_vid)
1723                 pstate_funcs.get_vid(cpu);
1724
1725         intel_pstate_set_min_pstate(cpu);
1726 }
1727
1728 static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
1729 {
1730         struct sample *sample = &cpu->sample;
1731
1732         sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
1733 }
1734
1735 static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
1736 {
1737         u64 aperf, mperf;
1738         unsigned long flags;
1739         u64 tsc;
1740
1741         local_irq_save(flags);
1742         rdmsrl(MSR_IA32_APERF, aperf);
1743         rdmsrl(MSR_IA32_MPERF, mperf);
1744         tsc = rdtsc();
1745         if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
1746                 local_irq_restore(flags);
1747                 return false;
1748         }
1749         local_irq_restore(flags);
1750
1751         cpu->last_sample_time = cpu->sample.time;
1752         cpu->sample.time = time;
1753         cpu->sample.aperf = aperf;
1754         cpu->sample.mperf = mperf;
1755         cpu->sample.tsc =  tsc;
1756         cpu->sample.aperf -= cpu->prev_aperf;
1757         cpu->sample.mperf -= cpu->prev_mperf;
1758         cpu->sample.tsc -= cpu->prev_tsc;
1759
1760         cpu->prev_aperf = aperf;
1761         cpu->prev_mperf = mperf;
1762         cpu->prev_tsc = tsc;
1763         /*
1764          * First time this function is invoked in a given cycle, all of the
1765          * previous sample data fields are equal to zero or stale and they must
1766          * be populated with meaningful numbers for things to work, so assume
1767          * that sample.time will always be reset before setting the utilization
1768          * update hook and make the caller skip the sample then.
1769          */
1770         return !!cpu->last_sample_time;
1771 }
1772
1773 static inline int32_t get_avg_frequency(struct cpudata *cpu)
1774 {
1775         return mul_ext_fp(cpu->sample.core_avg_perf,
1776                           cpu->pstate.max_pstate_physical * cpu->pstate.scaling);
1777 }
1778
1779 static inline int32_t get_avg_pstate(struct cpudata *cpu)
1780 {
1781         return mul_ext_fp(cpu->pstate.max_pstate_physical,
1782                           cpu->sample.core_avg_perf);
1783 }
1784
1785 static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu)
1786 {
1787         struct sample *sample = &cpu->sample;
1788         int32_t busy_frac, boost;
1789         int target, avg_pstate;
1790
1791         busy_frac = div_fp(sample->mperf, sample->tsc);
1792
1793         boost = cpu->iowait_boost;
1794         cpu->iowait_boost >>= 1;
1795
1796         if (busy_frac < boost)
1797                 busy_frac = boost;
1798
1799         sample->busy_scaled = busy_frac * 100;
1800
1801         target = global.no_turbo || global.turbo_disabled ?
1802                         cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
1803         target += target >> 2;
1804         target = mul_fp(target, busy_frac);
1805         if (target < cpu->pstate.min_pstate)
1806                 target = cpu->pstate.min_pstate;
1807
1808         /*
1809          * If the average P-state during the previous cycle was higher than the
1810          * current target, add 50% of the difference to the target to reduce
1811          * possible performance oscillations and offset possible performance
1812          * loss related to moving the workload from one CPU to another within
1813          * a package/module.
1814          */
1815         avg_pstate = get_avg_pstate(cpu);
1816         if (avg_pstate > target)
1817                 target += (avg_pstate - target) >> 1;
1818
1819         return target;
1820 }
1821
1822 static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu)
1823 {
1824         int32_t perf_scaled, max_pstate, current_pstate, sample_ratio;
1825         u64 duration_ns;
1826
1827         /*
1828          * perf_scaled is the ratio of the average P-state during the last
1829          * sampling period to the P-state requested last time (in percent).
1830          *
1831          * That measures the system's response to the previous P-state
1832          * selection.
1833          */
1834         max_pstate = cpu->pstate.max_pstate_physical;
1835         current_pstate = cpu->pstate.current_pstate;
1836         perf_scaled = mul_ext_fp(cpu->sample.core_avg_perf,
1837                                div_fp(100 * max_pstate, current_pstate));
1838
1839         /*
1840          * Since our utilization update callback will not run unless we are
1841          * in C0, check if the actual elapsed time is significantly greater (3x)
1842          * than our sample interval.  If it is, then we were idle for a long
1843          * enough period of time to adjust our performance metric.
1844          */
1845         duration_ns = cpu->sample.time - cpu->last_sample_time;
1846         if ((s64)duration_ns > pid_params.sample_rate_ns * 3) {
1847                 sample_ratio = div_fp(pid_params.sample_rate_ns, duration_ns);
1848                 perf_scaled = mul_fp(perf_scaled, sample_ratio);
1849         } else {
1850                 sample_ratio = div_fp(100 * cpu->sample.mperf, cpu->sample.tsc);
1851                 if (sample_ratio < int_tofp(1))
1852                         perf_scaled = 0;
1853         }
1854
1855         cpu->sample.busy_scaled = perf_scaled;
1856         return cpu->pstate.current_pstate - pid_calc(&cpu->pid, perf_scaled);
1857 }
1858
1859 static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
1860 {
1861         int max_perf, min_perf;
1862
1863         intel_pstate_get_min_max(cpu, &min_perf, &max_perf);
1864         pstate = clamp_t(int, pstate, min_perf, max_perf);
1865         return pstate;
1866 }
1867
1868 static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
1869 {
1870         if (pstate == cpu->pstate.current_pstate)
1871                 return;
1872
1873         cpu->pstate.current_pstate = pstate;
1874         wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
1875 }
1876
1877 static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu)
1878 {
1879         int from, target_pstate;
1880         struct sample *sample;
1881
1882         from = cpu->pstate.current_pstate;
1883
1884         target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ?
1885                 cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu);
1886
1887         update_turbo_state();
1888
1889         target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
1890         trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
1891         intel_pstate_update_pstate(cpu, target_pstate);
1892
1893         sample = &cpu->sample;
1894         trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
1895                 fp_toint(sample->busy_scaled),
1896                 from,
1897                 cpu->pstate.current_pstate,
1898                 sample->mperf,
1899                 sample->aperf,
1900                 sample->tsc,
1901                 get_avg_frequency(cpu),
1902                 fp_toint(cpu->iowait_boost * 100));
1903 }
1904
1905 static void intel_pstate_update_util(struct update_util_data *data, u64 time,
1906                                      unsigned int flags)
1907 {
1908         struct cpudata *cpu = container_of(data, struct cpudata, update_util);
1909         u64 delta_ns;
1910
1911         if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) {
1912                 if (flags & SCHED_CPUFREQ_IOWAIT) {
1913                         cpu->iowait_boost = int_tofp(1);
1914                 } else if (cpu->iowait_boost) {
1915                         /* Clear iowait_boost if the CPU may have been idle. */
1916                         delta_ns = time - cpu->last_update;
1917                         if (delta_ns > TICK_NSEC)
1918                                 cpu->iowait_boost = 0;
1919                 }
1920                 cpu->last_update = time;
1921         }
1922
1923         delta_ns = time - cpu->sample.time;
1924         if ((s64)delta_ns >= pid_params.sample_rate_ns) {
1925                 bool sample_taken = intel_pstate_sample(cpu, time);
1926
1927                 if (sample_taken) {
1928                         intel_pstate_calc_avg_perf(cpu);
1929                         if (!hwp_active)
1930                                 intel_pstate_adjust_busy_pstate(cpu);
1931                 }
1932         }
1933 }
1934
1935 #define ICPU(model, policy) \
1936         { X86_VENDOR_INTEL, 6, model, X86_FEATURE_APERFMPERF,\
1937                         (unsigned long)&policy }
1938
1939 static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
1940         ICPU(INTEL_FAM6_SANDYBRIDGE,            core_params),
1941         ICPU(INTEL_FAM6_SANDYBRIDGE_X,          core_params),
1942         ICPU(INTEL_FAM6_ATOM_SILVERMONT1,       silvermont_params),
1943         ICPU(INTEL_FAM6_IVYBRIDGE,              core_params),
1944         ICPU(INTEL_FAM6_HASWELL_CORE,           core_params),
1945         ICPU(INTEL_FAM6_BROADWELL_CORE,         core_params),
1946         ICPU(INTEL_FAM6_IVYBRIDGE_X,            core_params),
1947         ICPU(INTEL_FAM6_HASWELL_X,              core_params),
1948         ICPU(INTEL_FAM6_HASWELL_ULT,            core_params),
1949         ICPU(INTEL_FAM6_HASWELL_GT3E,           core_params),
1950         ICPU(INTEL_FAM6_BROADWELL_GT3E,         core_params),
1951         ICPU(INTEL_FAM6_ATOM_AIRMONT,           airmont_params),
1952         ICPU(INTEL_FAM6_SKYLAKE_MOBILE,         core_params),
1953         ICPU(INTEL_FAM6_BROADWELL_X,            core_params),
1954         ICPU(INTEL_FAM6_SKYLAKE_DESKTOP,        core_params),
1955         ICPU(INTEL_FAM6_BROADWELL_XEON_D,       core_params),
1956         ICPU(INTEL_FAM6_XEON_PHI_KNL,           knl_params),
1957         ICPU(INTEL_FAM6_XEON_PHI_KNM,           knl_params),
1958         ICPU(INTEL_FAM6_ATOM_GOLDMONT,          bxt_params),
1959         {}
1960 };
1961 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
1962
1963 static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
1964         ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params),
1965         ICPU(INTEL_FAM6_BROADWELL_X, core_params),
1966         ICPU(INTEL_FAM6_SKYLAKE_X, core_params),
1967         {}
1968 };
1969
1970 static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
1971         ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, core_params),
1972         {}
1973 };
1974
1975 static int intel_pstate_init_cpu(unsigned int cpunum)
1976 {
1977         struct cpudata *cpu;
1978
1979         cpu = all_cpu_data[cpunum];
1980
1981         if (!cpu) {
1982                 unsigned int size = sizeof(struct cpudata);
1983
1984                 if (per_cpu_limits)
1985                         size += sizeof(struct perf_limits);
1986
1987                 cpu = kzalloc(size, GFP_KERNEL);
1988                 if (!cpu)
1989                         return -ENOMEM;
1990
1991                 all_cpu_data[cpunum] = cpu;
1992                 if (per_cpu_limits)
1993                         cpu->perf_limits = (struct perf_limits *)(cpu + 1);
1994
1995                 cpu->epp_default = -EINVAL;
1996                 cpu->epp_powersave = -EINVAL;
1997                 cpu->epp_saved = -EINVAL;
1998         }
1999
2000         cpu = all_cpu_data[cpunum];
2001
2002         cpu->cpu = cpunum;
2003
2004         if (hwp_active) {
2005                 const struct x86_cpu_id *id;
2006
2007                 id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
2008                 if (id)
2009                         intel_pstate_disable_ee(cpunum);
2010
2011                 intel_pstate_hwp_enable(cpu);
2012                 pid_params.sample_rate_ms = 50;
2013                 pid_params.sample_rate_ns = 50 * NSEC_PER_MSEC;
2014         }
2015
2016         intel_pstate_get_cpu_pstates(cpu);
2017
2018         intel_pstate_busy_pid_reset(cpu);
2019
2020         pr_debug("controlling: cpu %d\n", cpunum);
2021
2022         return 0;
2023 }
2024
2025 static unsigned int intel_pstate_get(unsigned int cpu_num)
2026 {
2027         struct cpudata *cpu = all_cpu_data[cpu_num];
2028
2029         return cpu ? get_avg_frequency(cpu) : 0;
2030 }
2031
2032 static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
2033 {
2034         struct cpudata *cpu = all_cpu_data[cpu_num];
2035
2036         if (cpu->update_util_set)
2037                 return;
2038
2039         /* Prevent intel_pstate_update_util() from using stale data. */
2040         cpu->sample.time = 0;
2041         cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
2042                                      intel_pstate_update_util);
2043         cpu->update_util_set = true;
2044 }
2045
2046 static void intel_pstate_clear_update_util_hook(unsigned int cpu)
2047 {
2048         struct cpudata *cpu_data = all_cpu_data[cpu];
2049
2050         if (!cpu_data->update_util_set)
2051                 return;
2052
2053         cpufreq_remove_update_util_hook(cpu);
2054         cpu_data->update_util_set = false;
2055         synchronize_sched();
2056 }
2057
2058 static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy,
2059                                             struct perf_limits *limits)
2060 {
2061         int32_t max_policy_perf, min_policy_perf;
2062
2063         max_policy_perf = div_ext_fp(policy->max, policy->cpuinfo.max_freq);
2064         max_policy_perf = clamp_t(int32_t, max_policy_perf, 0, int_ext_tofp(1));
2065         if (policy->max == policy->min) {
2066                 min_policy_perf = max_policy_perf;
2067         } else {
2068                 min_policy_perf = div_ext_fp(policy->min,
2069                                              policy->cpuinfo.max_freq);
2070                 min_policy_perf = clamp_t(int32_t, min_policy_perf,
2071                                           0, max_policy_perf);
2072         }
2073
2074         /* Normalize user input to [min_perf, max_perf] */
2075         limits->min_perf = max(min_policy_perf,
2076                                percent_ext_fp(limits->min_sysfs_pct));
2077         limits->min_perf = min(limits->min_perf, max_policy_perf);
2078         limits->max_perf = min(max_policy_perf,
2079                                percent_ext_fp(limits->max_sysfs_pct));
2080         limits->max_perf = max(min_policy_perf, limits->max_perf);
2081
2082         /* Make sure min_perf <= max_perf */
2083         limits->min_perf = min(limits->min_perf, limits->max_perf);
2084
2085         limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS);
2086         limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS);
2087         limits->max_perf_pct = fp_ext_toint(limits->max_perf * 100);
2088         limits->min_perf_pct = fp_ext_toint(limits->min_perf * 100);
2089
2090         pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu,
2091                  limits->max_perf_pct, limits->min_perf_pct);
2092 }
2093
2094 static int intel_pstate_set_policy(struct cpufreq_policy *policy)
2095 {
2096         struct cpudata *cpu;
2097         struct perf_limits *perf_limits = &global;
2098
2099         if (!policy->cpuinfo.max_freq)
2100                 return -ENODEV;
2101
2102         pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
2103                  policy->cpuinfo.max_freq, policy->max);
2104
2105         cpu = all_cpu_data[policy->cpu];
2106         cpu->policy = policy->policy;
2107
2108         if (cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
2109             policy->max < policy->cpuinfo.max_freq &&
2110             policy->max > cpu->pstate.max_pstate * cpu->pstate.scaling) {
2111                 pr_debug("policy->max > max non turbo frequency\n");
2112                 policy->max = policy->cpuinfo.max_freq;
2113         }
2114
2115         if (per_cpu_limits)
2116                 perf_limits = cpu->perf_limits;
2117
2118         mutex_lock(&intel_pstate_limits_lock);
2119
2120         intel_pstate_update_perf_limits(policy, perf_limits);
2121
2122         if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
2123                 /*
2124                  * NOHZ_FULL CPUs need this as the governor callback may not
2125                  * be invoked on them.
2126                  */
2127                 intel_pstate_clear_update_util_hook(policy->cpu);
2128                 intel_pstate_max_within_limits(cpu);
2129         }
2130
2131         intel_pstate_set_update_util_hook(policy->cpu);
2132
2133         intel_pstate_hwp_set_policy(policy);
2134
2135         mutex_unlock(&intel_pstate_limits_lock);
2136
2137         return 0;
2138 }
2139
2140 static int intel_pstate_verify_policy(struct cpufreq_policy *policy)
2141 {
2142         struct cpudata *cpu = all_cpu_data[policy->cpu];
2143
2144         update_turbo_state();
2145         policy->cpuinfo.max_freq = global.turbo_disabled || global.no_turbo ?
2146                                         cpu->pstate.max_freq :
2147                                         cpu->pstate.turbo_freq;
2148
2149         cpufreq_verify_within_cpu_limits(policy);
2150
2151         if (policy->policy != CPUFREQ_POLICY_POWERSAVE &&
2152             policy->policy != CPUFREQ_POLICY_PERFORMANCE)
2153                 return -EINVAL;
2154
2155         /* When per-CPU limits are used, sysfs limits are not used */
2156         if (!per_cpu_limits) {
2157                 unsigned int max_freq, min_freq;
2158
2159                 max_freq = policy->cpuinfo.max_freq *
2160                                         global.max_sysfs_pct / 100;
2161                 min_freq = policy->cpuinfo.max_freq *
2162                                         global.min_sysfs_pct / 100;
2163                 cpufreq_verify_within_limits(policy, min_freq, max_freq);
2164         }
2165
2166         return 0;
2167 }
2168
2169 static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy)
2170 {
2171         intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]);
2172 }
2173
2174 static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
2175 {
2176         pr_debug("CPU %d exiting\n", policy->cpu);
2177
2178         intel_pstate_clear_update_util_hook(policy->cpu);
2179         if (hwp_active)
2180                 intel_pstate_hwp_save_state(policy);
2181         else
2182                 intel_cpufreq_stop_cpu(policy);
2183 }
2184
2185 static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
2186 {
2187         intel_pstate_exit_perf_limits(policy);
2188
2189         policy->fast_switch_possible = false;
2190
2191         return 0;
2192 }
2193
2194 static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
2195 {
2196         struct cpudata *cpu;
2197         int rc;
2198
2199         rc = intel_pstate_init_cpu(policy->cpu);
2200         if (rc)
2201                 return rc;
2202
2203         cpu = all_cpu_data[policy->cpu];
2204
2205         if (per_cpu_limits)
2206                 intel_pstate_init_limits(cpu->perf_limits);
2207
2208         policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
2209         policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
2210
2211         /* cpuinfo and default policy values */
2212         policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
2213         update_turbo_state();
2214         policy->cpuinfo.max_freq = global.turbo_disabled ?
2215                         cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
2216         policy->cpuinfo.max_freq *= cpu->pstate.scaling;
2217
2218         intel_pstate_init_acpi_perf_limits(policy);
2219         cpumask_set_cpu(policy->cpu, policy->cpus);
2220
2221         policy->fast_switch_possible = true;
2222
2223         return 0;
2224 }
2225
2226 static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
2227 {
2228         int ret = __intel_pstate_cpu_init(policy);
2229
2230         if (ret)
2231                 return ret;
2232
2233         policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL;
2234         if (IS_ENABLED(CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE))
2235                 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
2236         else
2237                 policy->policy = CPUFREQ_POLICY_POWERSAVE;
2238
2239         return 0;
2240 }
2241
2242 static struct cpufreq_driver intel_pstate = {
2243         .flags          = CPUFREQ_CONST_LOOPS,
2244         .verify         = intel_pstate_verify_policy,
2245         .setpolicy      = intel_pstate_set_policy,
2246         .suspend        = intel_pstate_hwp_save_state,
2247         .resume         = intel_pstate_resume,
2248         .get            = intel_pstate_get,
2249         .init           = intel_pstate_cpu_init,
2250         .exit           = intel_pstate_cpu_exit,
2251         .stop_cpu       = intel_pstate_stop_cpu,
2252         .name           = "intel_pstate",
2253 };
2254
2255 static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy)
2256 {
2257         struct cpudata *cpu = all_cpu_data[policy->cpu];
2258
2259         update_turbo_state();
2260         policy->cpuinfo.max_freq = global.no_turbo || global.turbo_disabled ?
2261                         cpu->pstate.max_freq : cpu->pstate.turbo_freq;
2262
2263         cpufreq_verify_within_cpu_limits(policy);
2264
2265         return 0;
2266 }
2267
2268 static int intel_cpufreq_target(struct cpufreq_policy *policy,
2269                                 unsigned int target_freq,
2270                                 unsigned int relation)
2271 {
2272         struct cpudata *cpu = all_cpu_data[policy->cpu];
2273         struct cpufreq_freqs freqs;
2274         int target_pstate;
2275
2276         update_turbo_state();
2277
2278         freqs.old = policy->cur;
2279         freqs.new = target_freq;
2280
2281         cpufreq_freq_transition_begin(policy, &freqs);
2282         switch (relation) {
2283         case CPUFREQ_RELATION_L:
2284                 target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling);
2285                 break;
2286         case CPUFREQ_RELATION_H:
2287                 target_pstate = freqs.new / cpu->pstate.scaling;
2288                 break;
2289         default:
2290                 target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling);
2291                 break;
2292         }
2293         target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2294         if (target_pstate != cpu->pstate.current_pstate) {
2295                 cpu->pstate.current_pstate = target_pstate;
2296                 wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL,
2297                               pstate_funcs.get_val(cpu, target_pstate));
2298         }
2299         freqs.new = target_pstate * cpu->pstate.scaling;
2300         cpufreq_freq_transition_end(policy, &freqs, false);
2301
2302         return 0;
2303 }
2304
2305 static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
2306                                               unsigned int target_freq)
2307 {
2308         struct cpudata *cpu = all_cpu_data[policy->cpu];
2309         int target_pstate;
2310
2311         update_turbo_state();
2312
2313         target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
2314         target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
2315         intel_pstate_update_pstate(cpu, target_pstate);
2316         return target_pstate * cpu->pstate.scaling;
2317 }
2318
2319 static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
2320 {
2321         int ret = __intel_pstate_cpu_init(policy);
2322
2323         if (ret)
2324                 return ret;
2325
2326         policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
2327         /* This reflects the intel_pstate_get_cpu_pstates() setting. */
2328         policy->cur = policy->cpuinfo.min_freq;
2329
2330         return 0;
2331 }
2332
2333 static struct cpufreq_driver intel_cpufreq = {
2334         .flags          = CPUFREQ_CONST_LOOPS,
2335         .verify         = intel_cpufreq_verify_policy,
2336         .target         = intel_cpufreq_target,
2337         .fast_switch    = intel_cpufreq_fast_switch,
2338         .init           = intel_cpufreq_cpu_init,
2339         .exit           = intel_pstate_cpu_exit,
2340         .stop_cpu       = intel_cpufreq_stop_cpu,
2341         .name           = "intel_cpufreq",
2342 };
2343
2344 static struct cpufreq_driver *intel_pstate_driver = &intel_pstate;
2345
2346 static void intel_pstate_driver_cleanup(void)
2347 {
2348         unsigned int cpu;
2349
2350         get_online_cpus();
2351         for_each_online_cpu(cpu) {
2352                 if (all_cpu_data[cpu]) {
2353                         if (intel_pstate_driver == &intel_pstate)
2354                                 intel_pstate_clear_update_util_hook(cpu);
2355
2356                         kfree(all_cpu_data[cpu]);
2357                         all_cpu_data[cpu] = NULL;
2358                 }
2359         }
2360         put_online_cpus();
2361 }
2362
2363 static int intel_pstate_register_driver(void)
2364 {
2365         int ret;
2366
2367         intel_pstate_init_limits(&global);
2368
2369         ret = cpufreq_register_driver(intel_pstate_driver);
2370         if (ret) {
2371                 intel_pstate_driver_cleanup();
2372                 return ret;
2373         }
2374
2375         mutex_lock(&intel_pstate_limits_lock);
2376         driver_registered = true;
2377         mutex_unlock(&intel_pstate_limits_lock);
2378
2379         if (intel_pstate_driver == &intel_pstate && !hwp_active &&
2380             pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
2381                 intel_pstate_debug_expose_params();
2382
2383         return 0;
2384 }
2385
2386 static int intel_pstate_unregister_driver(void)
2387 {
2388         if (hwp_active)
2389                 return -EBUSY;
2390
2391         if (intel_pstate_driver == &intel_pstate && !hwp_active &&
2392             pstate_funcs.get_target_pstate != get_target_pstate_use_cpu_load)
2393                 intel_pstate_debug_hide_params();
2394
2395         mutex_lock(&intel_pstate_limits_lock);
2396         driver_registered = false;
2397         mutex_unlock(&intel_pstate_limits_lock);
2398
2399         cpufreq_unregister_driver(intel_pstate_driver);
2400         intel_pstate_driver_cleanup();
2401
2402         return 0;
2403 }
2404
2405 static ssize_t intel_pstate_show_status(char *buf)
2406 {
2407         if (!driver_registered)
2408                 return sprintf(buf, "off\n");
2409
2410         return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
2411                                         "active" : "passive");
2412 }
2413
2414 static int intel_pstate_update_status(const char *buf, size_t size)
2415 {
2416         int ret;
2417
2418         if (size == 3 && !strncmp(buf, "off", size))
2419                 return driver_registered ?
2420                         intel_pstate_unregister_driver() : -EINVAL;
2421
2422         if (size == 6 && !strncmp(buf, "active", size)) {
2423                 if (driver_registered) {
2424                         if (intel_pstate_driver == &intel_pstate)
2425                                 return 0;
2426
2427                         ret = intel_pstate_unregister_driver();
2428                         if (ret)
2429                                 return ret;
2430                 }
2431
2432                 intel_pstate_driver = &intel_pstate;
2433                 return intel_pstate_register_driver();
2434         }
2435
2436         if (size == 7 && !strncmp(buf, "passive", size)) {
2437                 if (driver_registered) {
2438                         if (intel_pstate_driver != &intel_pstate)
2439                                 return 0;
2440
2441                         ret = intel_pstate_unregister_driver();
2442                         if (ret)
2443                                 return ret;
2444                 }
2445
2446                 intel_pstate_driver = &intel_cpufreq;
2447                 return intel_pstate_register_driver();
2448         }
2449
2450         return -EINVAL;
2451 }
2452
2453 static int no_load __initdata;
2454 static int no_hwp __initdata;
2455 static int hwp_only __initdata;
2456 static unsigned int force_load __initdata;
2457
2458 static int __init intel_pstate_msrs_not_valid(void)
2459 {
2460         if (!pstate_funcs.get_max() ||
2461             !pstate_funcs.get_min() ||
2462             !pstate_funcs.get_turbo())
2463                 return -ENODEV;
2464
2465         return 0;
2466 }
2467
2468 static void __init copy_pid_params(struct pstate_adjust_policy *policy)
2469 {
2470         pid_params.sample_rate_ms = policy->sample_rate_ms;
2471         pid_params.sample_rate_ns = pid_params.sample_rate_ms * NSEC_PER_MSEC;
2472         pid_params.p_gain_pct = policy->p_gain_pct;
2473         pid_params.i_gain_pct = policy->i_gain_pct;
2474         pid_params.d_gain_pct = policy->d_gain_pct;
2475         pid_params.deadband = policy->deadband;
2476         pid_params.setpoint = policy->setpoint;
2477 }
2478
2479 #ifdef CONFIG_ACPI
2480 static void intel_pstate_use_acpi_profile(void)
2481 {
2482         if (acpi_gbl_FADT.preferred_profile == PM_MOBILE)
2483                 pstate_funcs.get_target_pstate =
2484                                 get_target_pstate_use_cpu_load;
2485 }
2486 #else
2487 static void intel_pstate_use_acpi_profile(void)
2488 {
2489 }
2490 #endif
2491
2492 static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
2493 {
2494         pstate_funcs.get_max   = funcs->get_max;
2495         pstate_funcs.get_max_physical = funcs->get_max_physical;
2496         pstate_funcs.get_min   = funcs->get_min;
2497         pstate_funcs.get_turbo = funcs->get_turbo;
2498         pstate_funcs.get_scaling = funcs->get_scaling;
2499         pstate_funcs.get_val   = funcs->get_val;
2500         pstate_funcs.get_vid   = funcs->get_vid;
2501         pstate_funcs.get_target_pstate = funcs->get_target_pstate;
2502
2503         intel_pstate_use_acpi_profile();
2504 }
2505
2506 #ifdef CONFIG_ACPI
2507
2508 static bool __init intel_pstate_no_acpi_pss(void)
2509 {
2510         int i;
2511
2512         for_each_possible_cpu(i) {
2513                 acpi_status status;
2514                 union acpi_object *pss;
2515                 struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
2516                 struct acpi_processor *pr = per_cpu(processors, i);
2517
2518                 if (!pr)
2519                         continue;
2520
2521                 status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
2522                 if (ACPI_FAILURE(status))
2523                         continue;
2524
2525                 pss = buffer.pointer;
2526                 if (pss && pss->type == ACPI_TYPE_PACKAGE) {
2527                         kfree(pss);
2528                         return false;
2529                 }
2530
2531                 kfree(pss);
2532         }
2533
2534         return true;
2535 }
2536
2537 static bool __init intel_pstate_has_acpi_ppc(void)
2538 {
2539         int i;
2540
2541         for_each_possible_cpu(i) {
2542                 struct acpi_processor *pr = per_cpu(processors, i);
2543
2544                 if (!pr)
2545                         continue;
2546                 if (acpi_has_method(pr->handle, "_PPC"))
2547                         return true;
2548         }
2549         return false;
2550 }
2551
2552 enum {
2553         PSS,
2554         PPC,
2555 };
2556
2557 struct hw_vendor_info {
2558         u16  valid;
2559         char oem_id[ACPI_OEM_ID_SIZE];
2560         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE];
2561         int  oem_pwr_table;
2562 };
2563
2564 /* Hardware vendor-specific info that has its own power management modes */
2565 static struct hw_vendor_info vendor_info[] __initdata = {
2566         {1, "HP    ", "ProLiant", PSS},
2567         {1, "ORACLE", "X4-2    ", PPC},
2568         {1, "ORACLE", "X4-2L   ", PPC},
2569         {1, "ORACLE", "X4-2B   ", PPC},
2570         {1, "ORACLE", "X3-2    ", PPC},
2571         {1, "ORACLE", "X3-2L   ", PPC},
2572         {1, "ORACLE", "X3-2B   ", PPC},
2573         {1, "ORACLE", "X4470M2 ", PPC},
2574         {1, "ORACLE", "X4270M3 ", PPC},
2575         {1, "ORACLE", "X4270M2 ", PPC},
2576         {1, "ORACLE", "X4170M2 ", PPC},
2577         {1, "ORACLE", "X4170 M3", PPC},
2578         {1, "ORACLE", "X4275 M3", PPC},
2579         {1, "ORACLE", "X6-2    ", PPC},
2580         {1, "ORACLE", "Sudbury ", PPC},
2581         {0, "", ""},
2582 };
2583
2584 static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
2585 {
2586         struct acpi_table_header hdr;
2587         struct hw_vendor_info *v_info;
2588         const struct x86_cpu_id *id;
2589         u64 misc_pwr;
2590
2591         id = x86_match_cpu(intel_pstate_cpu_oob_ids);
2592         if (id) {
2593                 rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
2594                 if ( misc_pwr & (1 << 8))
2595                         return true;
2596         }
2597
2598         if (acpi_disabled ||
2599             ACPI_FAILURE(acpi_get_table_header(ACPI_SIG_FADT, 0, &hdr)))
2600                 return false;
2601
2602         for (v_info = vendor_info; v_info->valid; v_info++) {
2603                 if (!strncmp(hdr.oem_id, v_info->oem_id, ACPI_OEM_ID_SIZE) &&
2604                         !strncmp(hdr.oem_table_id, v_info->oem_table_id,
2605                                                 ACPI_OEM_TABLE_ID_SIZE))
2606                         switch (v_info->oem_pwr_table) {
2607                         case PSS:
2608                                 return intel_pstate_no_acpi_pss();
2609                         case PPC:
2610                                 return intel_pstate_has_acpi_ppc() &&
2611                                         (!force_load);
2612                         }
2613         }
2614
2615         return false;
2616 }
2617
2618 static void intel_pstate_request_control_from_smm(void)
2619 {
2620         /*
2621          * It may be unsafe to request P-states control from SMM if _PPC support
2622          * has not been enabled.
2623          */
2624         if (acpi_ppc)
2625                 acpi_processor_pstate_control();
2626 }
2627 #else /* CONFIG_ACPI not enabled */
2628 static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
2629 static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
2630 static inline void intel_pstate_request_control_from_smm(void) {}
2631 #endif /* CONFIG_ACPI */
2632
2633 static const struct x86_cpu_id hwp_support_ids[] __initconst = {
2634         { X86_VENDOR_INTEL, 6, X86_MODEL_ANY, X86_FEATURE_HWP },
2635         {}
2636 };
2637
2638 static int __init intel_pstate_init(void)
2639 {
2640         const struct x86_cpu_id *id;
2641         struct cpu_defaults *cpu_def;
2642         int rc = 0;
2643
2644         if (no_load)
2645                 return -ENODEV;
2646
2647         if (x86_match_cpu(hwp_support_ids) && !no_hwp) {
2648                 copy_cpu_funcs(&core_params.funcs);
2649                 hwp_active++;
2650                 intel_pstate.attr = hwp_cpufreq_attrs;
2651                 goto hwp_cpu_matched;
2652         }
2653
2654         id = x86_match_cpu(intel_pstate_cpu_ids);
2655         if (!id)
2656                 return -ENODEV;
2657
2658         cpu_def = (struct cpu_defaults *)id->driver_data;
2659
2660         copy_pid_params(&cpu_def->pid_policy);
2661         copy_cpu_funcs(&cpu_def->funcs);
2662
2663         if (intel_pstate_msrs_not_valid())
2664                 return -ENODEV;
2665
2666 hwp_cpu_matched:
2667         /*
2668          * The Intel pstate driver will be ignored if the platform
2669          * firmware has its own power management modes.
2670          */
2671         if (intel_pstate_platform_pwr_mgmt_exists())
2672                 return -ENODEV;
2673
2674         if (!hwp_active && hwp_only)
2675                 return -ENOTSUPP;
2676
2677         pr_info("Intel P-state driver initializing\n");
2678
2679         all_cpu_data = vzalloc(sizeof(void *) * num_possible_cpus());
2680         if (!all_cpu_data)
2681                 return -ENOMEM;
2682
2683         intel_pstate_request_control_from_smm();
2684
2685         intel_pstate_sysfs_expose_params();
2686
2687         mutex_lock(&intel_pstate_driver_lock);
2688         rc = intel_pstate_register_driver();
2689         mutex_unlock(&intel_pstate_driver_lock);
2690         if (rc)
2691                 return rc;
2692
2693         if (hwp_active)
2694                 pr_info("HWP enabled\n");
2695
2696         return 0;
2697 }
2698 device_initcall(intel_pstate_init);
2699
2700 static int __init intel_pstate_setup(char *str)
2701 {
2702         if (!str)
2703                 return -EINVAL;
2704
2705         if (!strcmp(str, "disable")) {
2706                 no_load = 1;
2707         } else if (!strcmp(str, "passive")) {
2708                 pr_info("Passive mode enabled\n");
2709                 intel_pstate_driver = &intel_cpufreq;
2710                 no_hwp = 1;
2711         }
2712         if (!strcmp(str, "no_hwp")) {
2713                 pr_info("HWP disabled\n");
2714                 no_hwp = 1;
2715         }
2716         if (!strcmp(str, "force"))
2717                 force_load = 1;
2718         if (!strcmp(str, "hwp_only"))
2719                 hwp_only = 1;
2720         if (!strcmp(str, "per_cpu_perf_limits"))
2721                 per_cpu_limits = true;
2722
2723 #ifdef CONFIG_ACPI
2724         if (!strcmp(str, "support_acpi_ppc"))
2725                 acpi_ppc = true;
2726 #endif
2727
2728         return 0;
2729 }
2730 early_param("intel_pstate", intel_pstate_setup);
2731
2732 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
2733 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
2734 MODULE_LICENSE("GPL");