drivers/cpufreq/powernv-cpufreq.c

   1 /*
   2  * POWERNV cpufreq driver for the IBM POWER processors
   3  *
   4  * (C) Copyright IBM 2014
   5  *
   6  * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2, or (at your option)
  11  * any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  */
  19
  20 #define pr_fmt(fmt)     "powernv-cpufreq: " fmt
  21
  22 #include <linux/kernel.h>
  23 #include <linux/sysfs.h>
  24 #include <linux/cpumask.h>
  25 #include <linux/module.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/smp.h>
  28 #include <linux/of.h>
  29 #include <linux/reboot.h>
  30 #include <linux/slab.h>
  31 #include <linux/cpu.h>
  32 #include <trace/events/power.h>
  33
  34 #include <asm/cputhreads.h>
  35 #include <asm/firmware.h>
  36 #include <asm/reg.h>
  37 #include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */
  38 #include <asm/opal.h>
  39
  40 #define POWERNV_MAX_PSTATES     256
  41 #define PMSR_PSAFE_ENABLE       (1UL << 30)
  42 #define PMSR_SPR_EM_DISABLE     (1UL << 31)
  43 #define PMSR_MAX(x)             ((x >> 32) & 0xFF)
  44
  45 static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
  46 static bool rebooting, throttled, occ_reset;
  47 static unsigned int *core_to_chip_map;
  48
  49 static const char * const throttle_reason[] = {
  50         "No throttling",
  51         "Power Cap",
  52         "Processor Over Temperature",
  53         "Power Supply Failure",
  54         "Over Current",
  55         "OCC Reset"
  56 };
  57
  58 static struct chip {
  59         unsigned int id;
  60         bool throttled;
  61         bool restore;
  62         u8 throttle_reason;
  63         cpumask_t mask;
  64         struct work_struct throttle;
  65 } *chips;
  66
  67 static int nr_chips;
  68
  69 /*
  70  * Note: The set of pstates consists of contiguous integers, the
  71  * smallest of which is indicated by powernv_pstate_info.min, the
  72  * largest of which is indicated by powernv_pstate_info.max.
  73  *
  74  * The nominal pstate is the highest non-turbo pstate in this
  75  * platform. This is indicated by powernv_pstate_info.nominal.
  76  */
  77 static struct powernv_pstate_info {
  78         int min;
  79         int max;
  80         int nominal;
  81         int nr_pstates;
  82 } powernv_pstate_info;
  83
  84 /*
  85  * Initialize the freq table based on data obtained
  86  * from the firmware passed via device-tree
  87  */
  88 static int init_powernv_pstates(void)
  89 {
  90         struct device_node *power_mgt;
  91         int i, pstate_min, pstate_max, pstate_nominal, nr_pstates = 0;
  92         const __be32 *pstate_ids, *pstate_freqs;
  93         u32 len_ids, len_freqs;
  94
  95         power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
  96         if (!power_mgt) {
  97                 pr_warn("power-mgt node not found\n");
  98                 return -ENODEV;
  99         }
 100
 101         if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) {
 102                 pr_warn("ibm,pstate-min node not found\n");
 103                 return -ENODEV;
 104         }
 105
 106         if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) {
 107                 pr_warn("ibm,pstate-max node not found\n");
 108                 return -ENODEV;
 109         }
 110
 111         if (of_property_read_u32(power_mgt, "ibm,pstate-nominal",
 112                                  &pstate_nominal)) {
 113                 pr_warn("ibm,pstate-nominal not found\n");
 114                 return -ENODEV;
 115         }
 116         pr_info("cpufreq pstate min %d nominal %d max %d\n", pstate_min,
 117                 pstate_nominal, pstate_max);
 118
 119         pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids);
 120         if (!pstate_ids) {
 121                 pr_warn("ibm,pstate-ids not found\n");
 122                 return -ENODEV;
 123         }
 124
 125         pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz",
 126                                       &len_freqs);
 127         if (!pstate_freqs) {
 128                 pr_warn("ibm,pstate-frequencies-mhz not found\n");
 129                 return -ENODEV;
 130         }
 131
 132         if (len_ids != len_freqs) {
 133                 pr_warn("Entries in ibm,pstate-ids and "
 134                         "ibm,pstate-frequencies-mhz does not match\n");
 135         }
 136
 137         nr_pstates = min(len_ids, len_freqs) / sizeof(u32);
 138         if (!nr_pstates) {
 139                 pr_warn("No PStates found\n");
 140                 return -ENODEV;
 141         }
 142
 143         pr_debug("NR PStates %d\n", nr_pstates);
 144         for (i = 0; i < nr_pstates; i++) {
 145                 u32 id = be32_to_cpu(pstate_ids[i]);
 146                 u32 freq = be32_to_cpu(pstate_freqs[i]);
 147
 148                 pr_debug("PState id %d freq %d MHz\n", id, freq);
 149                 powernv_freqs[i].frequency = freq * 1000; /* kHz */
 150                 powernv_freqs[i].driver_data = id;
 151         }
 152         /* End of list marker entry */
 153         powernv_freqs[i].frequency = CPUFREQ_TABLE_END;
 154
 155         powernv_pstate_info.min = pstate_min;
 156         powernv_pstate_info.max = pstate_max;
 157         powernv_pstate_info.nominal = pstate_nominal;
 158         powernv_pstate_info.nr_pstates = nr_pstates;
 159
 160         return 0;
 161 }
 162
 163 /* Returns the CPU frequency corresponding to the pstate_id. */
 164 static unsigned int pstate_id_to_freq(int pstate_id)
 165 {
 166         int i;
 167
 168         i = powernv_pstate_info.max - pstate_id;
 169         if (i >= powernv_pstate_info.nr_pstates || i < 0) {
 170                 pr_warn("PState id %d outside of PState table, "
 171                         "reporting nominal id %d instead\n",
 172                         pstate_id, powernv_pstate_info.nominal);
 173                 i = powernv_pstate_info.max - powernv_pstate_info.nominal;
 174         }
 175
 176         return powernv_freqs[i].frequency;
 177 }
 178
 179 /*
 180  * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by
 181  * the firmware
 182  */
 183 static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,
 184                                         char *buf)
 185 {
 186         return sprintf(buf, "%u\n",
 187                 pstate_id_to_freq(powernv_pstate_info.nominal));
 188 }
 189
 190 struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =
 191         __ATTR_RO(cpuinfo_nominal_freq);
 192
 193 static struct freq_attr *powernv_cpu_freq_attr[] = {
 194         &cpufreq_freq_attr_scaling_available_freqs,
 195         &cpufreq_freq_attr_cpuinfo_nominal_freq,
 196         NULL,
 197 };
 198
 199 /* Helper routines */
 200
 201 /* Access helpers to power mgt SPR */
 202
 203 static inline unsigned long get_pmspr(unsigned long sprn)
 204 {
 205         switch (sprn) {
 206         case SPRN_PMCR:
 207                 return mfspr(SPRN_PMCR);
 208
 209         case SPRN_PMICR:
 210                 return mfspr(SPRN_PMICR);
 211
 212         case SPRN_PMSR:
 213                 return mfspr(SPRN_PMSR);
 214         }
 215         BUG();
 216 }
 217
 218 static inline void set_pmspr(unsigned long sprn, unsigned long val)
 219 {
 220         switch (sprn) {
 221         case SPRN_PMCR:
 222                 mtspr(SPRN_PMCR, val);
 223                 return;
 224
 225         case SPRN_PMICR:
 226                 mtspr(SPRN_PMICR, val);
 227                 return;
 228         }
 229         BUG();
 230 }
 231
 232 /*
 233  * Use objects of this type to query/update
 234  * pstates on a remote CPU via smp_call_function.
 235  */
 236 struct powernv_smp_call_data {
 237         unsigned int freq;
 238         int pstate_id;
 239 };
 240
 241 /*
 242  * powernv_read_cpu_freq: Reads the current frequency on this CPU.
 243  *
 244  * Called via smp_call_function.
 245  *
 246  * Note: The caller of the smp_call_function should pass an argument of
 247  * the type 'struct powernv_smp_call_data *' along with this function.
 248  *
 249  * The current frequency on this CPU will be returned via
 250  * ((struct powernv_smp_call_data *)arg)->freq;
 251  */
 252 static void powernv_read_cpu_freq(void *arg)
 253 {
 254         unsigned long pmspr_val;
 255         s8 local_pstate_id;
 256         struct powernv_smp_call_data *freq_data = arg;
 257
 258         pmspr_val = get_pmspr(SPRN_PMSR);
 259
 260         /*
 261          * The local pstate id corresponds bits 48..55 in the PMSR.
 262          * Note: Watch out for the sign!
 263          */
 264         local_pstate_id = (pmspr_val >> 48) & 0xFF;
 265         freq_data->pstate_id = local_pstate_id;
 266         freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);
 267
 268         pr_debug("cpu %d pmsr %016lX pstate_id %d frequency %d kHz\n",
 269                 raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
 270                 freq_data->freq);
 271 }
 272
 273 /*
 274  * powernv_cpufreq_get: Returns the CPU frequency as reported by the
 275  * firmware for CPU 'cpu'. This value is reported through the sysfs
 276  * file cpuinfo_cur_freq.
 277  */
 278 static unsigned int powernv_cpufreq_get(unsigned int cpu)
 279 {
 280         struct powernv_smp_call_data freq_data;
 281
 282         smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq,
 283                         &freq_data, 1);
 284
 285         return freq_data.freq;
 286 }
 287
 288 /*
 289  * set_pstate: Sets the pstate on this CPU.
 290  *
 291  * This is called via an smp_call_function.
 292  *
 293  * The caller must ensure that freq_data is of the type
 294  * (struct powernv_smp_call_data *) and the pstate_id which needs to be set
 295  * on this CPU should be present in freq_data->pstate_id.
 296  */
 297 static void set_pstate(void *freq_data)
 298 {
 299         unsigned long val;
 300         unsigned long pstate_ul =
 301                 ((struct powernv_smp_call_data *) freq_data)->pstate_id;
 302
 303         val = get_pmspr(SPRN_PMCR);
 304         val = val & 0x0000FFFFFFFFFFFFULL;
 305
 306         pstate_ul = pstate_ul & 0xFF;
 307
 308         /* Set both global(bits 56..63) and local(bits 48..55) PStates */
 309         val = val | (pstate_ul << 56) | (pstate_ul << 48);
 310
 311         pr_debug("Setting cpu %d pmcr to %016lX\n",
 312                         raw_smp_processor_id(), val);
 313         set_pmspr(SPRN_PMCR, val);
 314 }
 315
 316 /*
 317  * get_nominal_index: Returns the index corresponding to the nominal
 318  * pstate in the cpufreq table
 319  */
 320 static inline unsigned int get_nominal_index(void)
 321 {
 322         return powernv_pstate_info.max - powernv_pstate_info.nominal;
 323 }
 324
 325 static void powernv_cpufreq_throttle_check(void *data)
 326 {
 327         unsigned int cpu = smp_processor_id();
 328         unsigned int chip_id = core_to_chip_map[cpu_core_index_of_thread(cpu)];
 329         unsigned long pmsr;
 330         int pmsr_pmax, i;
 331
 332         pmsr = get_pmspr(SPRN_PMSR);
 333
 334         for (i = 0; i < nr_chips; i++)
 335                 if (chips[i].id == chip_id)
 336                         break;
 337
 338         /* Check for Pmax Capping */
 339         pmsr_pmax = (s8)PMSR_MAX(pmsr);
 340         if (pmsr_pmax != powernv_pstate_info.max) {
 341                 if (chips[i].throttled)
 342                         goto next;
 343                 chips[i].throttled = true;
 344                 if (pmsr_pmax < powernv_pstate_info.nominal)
 345                         pr_warn_once("CPU %d on Chip %u has Pmax reduced below nominal frequency (%d < %d)\n",
 346                                      cpu, chips[i].id, pmsr_pmax,
 347                                      powernv_pstate_info.nominal);
 348                 trace_powernv_throttle(chips[i].id,
 349                                       throttle_reason[chips[i].throttle_reason],
 350                                       pmsr_pmax);
 351         } else if (chips[i].throttled) {
 352                 chips[i].throttled = false;
 353                 trace_powernv_throttle(chips[i].id,
 354                                       throttle_reason[chips[i].throttle_reason],
 355                                       pmsr_pmax);
 356         }
 357
 358         /* Check if Psafe_mode_active is set in PMSR. */
 359 next:
 360         if (pmsr & PMSR_PSAFE_ENABLE) {
 361                 throttled = true;
 362                 pr_info("Pstate set to safe frequency\n");
 363         }
 364
 365         /* Check if SPR_EM_DISABLE is set in PMSR */
 366         if (pmsr & PMSR_SPR_EM_DISABLE) {
 367                 throttled = true;
 368                 pr_info("Frequency Control disabled from OS\n");
 369         }
 370
 371         if (throttled) {
 372                 pr_info("PMSR = %16lx\n", pmsr);
 373                 pr_warn("CPU Frequency could be throttled\n");
 374         }
 375 }
 376
 377 /*
 378  * powernv_cpufreq_target_index: Sets the frequency corresponding to
 379  * the cpufreq table entry indexed by new_index on the cpus in the
 380  * mask policy->cpus
 381  */
 382 static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
 383                                         unsigned int new_index)
 384 {
 385         struct powernv_smp_call_data freq_data;
 386
 387         if (unlikely(rebooting) && new_index != get_nominal_index())
 388                 return 0;
 389
 390         if (!throttled)
 391                 powernv_cpufreq_throttle_check(NULL);
 392
 393         freq_data.pstate_id = powernv_freqs[new_index].driver_data;
 394
 395         /*
 396          * Use smp_call_function to send IPI and execute the
 397          * mtspr on target CPU.  We could do that without IPI
 398          * if current CPU is within policy->cpus (core)
 399          */
 400         smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
 401
 402         return 0;
 403 }
 404
 405 static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)
 406 {
 407         int base, i;
 408
 409         base = cpu_first_thread_sibling(policy->cpu);
 410
 411         for (i = 0; i < threads_per_core; i++)
 412                 cpumask_set_cpu(base + i, policy->cpus);
 413
 414         return cpufreq_table_validate_and_show(policy, powernv_freqs);
 415 }
 416
 417 static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb,
 418                                 unsigned long action, void *unused)
 419 {
 420         int cpu;
 421         struct cpufreq_policy cpu_policy;
 422
 423         rebooting = true;
 424         for_each_online_cpu(cpu) {
 425                 cpufreq_get_policy(&cpu_policy, cpu);
 426                 powernv_cpufreq_target_index(&cpu_policy, get_nominal_index());
 427         }
 428
 429         return NOTIFY_DONE;
 430 }
 431
 432 static struct notifier_block powernv_cpufreq_reboot_nb = {
 433         .notifier_call = powernv_cpufreq_reboot_notifier,
 434 };
 435
 436 void powernv_cpufreq_work_fn(struct work_struct *work)
 437 {
 438         struct chip *chip = container_of(work, struct chip, throttle);
 439         unsigned int cpu;
 440         cpumask_t mask;
 441
 442         get_online_cpus();
 443         cpumask_and(&mask, &chip->mask, cpu_online_mask);
 444         smp_call_function_any(&mask,
 445                               powernv_cpufreq_throttle_check, NULL, 0);
 446
 447         if (!chip->restore)
 448                 goto out;
 449
 450         chip->restore = false;
 451         for_each_cpu(cpu, &mask) {
 452                 int index;
 453                 struct cpufreq_policy policy;
 454
 455                 cpufreq_get_policy(&policy, cpu);
 456                 cpufreq_frequency_table_target(&policy, policy.freq_table,
 457                                                policy.cur,
 458                                                CPUFREQ_RELATION_C, &index);
 459                 powernv_cpufreq_target_index(&policy, index);
 460                 cpumask_andnot(&mask, &mask, policy.cpus);
 461         }
 462 out:
 463         put_online_cpus();
 464 }
 465
 466 static int powernv_cpufreq_occ_msg(struct notifier_block *nb,
 467                                    unsigned long msg_type, void *_msg)
 468 {
 469         struct opal_msg *msg = _msg;
 470         struct opal_occ_msg omsg;
 471         int i;
 472
 473         if (msg_type != OPAL_MSG_OCC)
 474                 return 0;
 475
 476         omsg.type = be64_to_cpu(msg->params[0]);
 477
 478         switch (omsg.type) {
 479         case OCC_RESET:
 480                 occ_reset = true;
 481                 pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");
 482                 /*
 483                  * powernv_cpufreq_throttle_check() is called in
 484                  * target() callback which can detect the throttle state
 485                  * for governors like ondemand.
 486                  * But static governors will not call target() often thus
 487                  * report throttling here.
 488                  */
 489                 if (!throttled) {
 490                         throttled = true;
 491                         pr_warn("CPU frequency is throttled for duration\n");
 492                 }
 493
 494                 break;
 495         case OCC_LOAD:
 496                 pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");
 497                 break;
 498         case OCC_THROTTLE:
 499                 omsg.chip = be64_to_cpu(msg->params[1]);
 500                 omsg.throttle_status = be64_to_cpu(msg->params[2]);
 501
 502                 if (occ_reset) {
 503                         occ_reset = false;
 504                         throttled = false;
 505                         pr_info("OCC Active, CPU frequency is no longer throttled\n");
 506
 507                         for (i = 0; i < nr_chips; i++) {
 508                                 chips[i].restore = true;
 509                                 schedule_work(&chips[i].throttle);
 510                         }
 511
 512                         return 0;
 513                 }
 514
 515                 for (i = 0; i < nr_chips; i++)
 516                         if (chips[i].id == omsg.chip)
 517                                 break;
 518
 519                 if (omsg.throttle_status >= 0 &&
 520                     omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS)
 521                         chips[i].throttle_reason = omsg.throttle_status;
 522
 523                 if (!omsg.throttle_status)
 524                         chips[i].restore = true;
 525
 526                 schedule_work(&chips[i].throttle);
 527         }
 528         return 0;
 529 }
 530
 531 static struct notifier_block powernv_cpufreq_opal_nb = {
 532         .notifier_call  = powernv_cpufreq_occ_msg,
 533         .next           = NULL,
 534         .priority       = 0,
 535 };
 536
 537 static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
 538 {
 539         struct powernv_smp_call_data freq_data;
 540
 541         freq_data.pstate_id = powernv_pstate_info.min;
 542         smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);
 543 }
 544
 545 static struct cpufreq_driver powernv_cpufreq_driver = {
 546         .name           = "powernv-cpufreq",
 547         .flags          = CPUFREQ_CONST_LOOPS,
 548         .init           = powernv_cpufreq_cpu_init,
 549         .verify         = cpufreq_generic_frequency_table_verify,
 550         .target_index   = powernv_cpufreq_target_index,
 551         .get            = powernv_cpufreq_get,
 552         .stop_cpu       = powernv_cpufreq_stop_cpu,
 553         .attr           = powernv_cpu_freq_attr,
 554 };
 555
 556 static int init_chip_info(void)
 557 {
 558         unsigned int chip[256];
 559         unsigned int cpu, i;
 560         unsigned int prev_chip_id = UINT_MAX;
 561         cpumask_t cpu_mask;
 562         int ret = -ENOMEM;
 563
 564         core_to_chip_map = kcalloc(cpu_nr_cores(), sizeof(unsigned int),
 565                                    GFP_KERNEL);
 566         if (!core_to_chip_map)
 567                 goto out;
 568
 569         cpumask_copy(&cpu_mask, cpu_possible_mask);
 570         for_each_cpu(cpu, &cpu_mask) {
 571                 unsigned int id = cpu_to_chip_id(cpu);
 572
 573                 if (prev_chip_id != id) {
 574                         prev_chip_id = id;
 575                         chip[nr_chips++] = id;
 576                 }
 577                 core_to_chip_map[cpu_core_index_of_thread(cpu)] = id;
 578                 cpumask_andnot(&cpu_mask, &cpu_mask, cpu_sibling_mask(cpu));
 579         }
 580
 581         chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL);
 582         if (!chips)
 583                 goto free_chip_map;
 584
 585         for (i = 0; i < nr_chips; i++) {
 586                 chips[i].id = chip[i];
 587                 cpumask_copy(&chips[i].mask, cpumask_of_node(chip[i]));
 588                 INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);
 589         }
 590
 591         return 0;
 592 free_chip_map:
 593         kfree(core_to_chip_map);
 594 out:
 595         return ret;
 596 }
 597
 598 static int __init powernv_cpufreq_init(void)
 599 {
 600         int rc = 0;
 601
 602         /* Don't probe on pseries (guest) platforms */
 603         if (!firmware_has_feature(FW_FEATURE_OPAL))
 604                 return -ENODEV;
 605
 606         /* Discover pstates from device tree and init */
 607         rc = init_powernv_pstates();
 608         if (rc) {
 609                 pr_info("powernv-cpufreq disabled. System does not support PState control\n");
 610                 return rc;
 611         }
 612
 613         /* Populate chip info */
 614         rc = init_chip_info();
 615         if (rc)
 616                 return rc;
 617
 618         register_reboot_notifier(&powernv_cpufreq_reboot_nb);
 619         opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);
 620         return cpufreq_register_driver(&powernv_cpufreq_driver);
 621 }
 622 module_init(powernv_cpufreq_init);
 623
 624 static void __exit powernv_cpufreq_exit(void)
 625 {
 626         unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);
 627         opal_message_notifier_unregister(OPAL_MSG_OCC,
 628                                          &powernv_cpufreq_opal_nb);
 629         kfree(chips);
 630         kfree(core_to_chip_map);
 631         cpufreq_unregister_driver(&powernv_cpufreq_driver);
 632 }
 633 module_exit(powernv_cpufreq_exit);
 634
 635 MODULE_LICENSE("GPL");
 636 MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>");