arch/x86/oprofile/nmi_int.c

   1 /**
   2  * @file nmi_int.c
   3  *
   4  * @remark Copyright 2002-2008 OProfile authors
   5  * @remark Read the file COPYING
   6  *
   7  * @author John Levon <levon@movementarian.org>
   8  * @author Robert Richter <robert.richter@amd.com>
   9  */
  10
  11 #include <linux/init.h>
  12 #include <linux/notifier.h>
  13 #include <linux/smp.h>
  14 #include <linux/oprofile.h>
  15 #include <linux/sysdev.h>
  16 #include <linux/slab.h>
  17 #include <linux/moduleparam.h>
  18 #include <linux/kdebug.h>
  19 #include <linux/cpu.h>
  20 #include <asm/nmi.h>
  21 #include <asm/msr.h>
  22 #include <asm/apic.h>
  23
  24 #include "op_counter.h"
  25 #include "op_x86_model.h"
  26
  27 DEFINE_PER_CPU(int, switch_index);
  28
  29 static struct op_x86_model_spec const *model;
  30 static DEFINE_PER_CPU(struct op_msrs, cpu_msrs);
  31 static DEFINE_PER_CPU(unsigned long, saved_lvtpc);
  32
  33 static int nmi_start(void);
  34 static void nmi_stop(void);
  35 static void nmi_cpu_start(void *dummy);
  36 static void nmi_cpu_stop(void *dummy);
  37 static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs);
  38 static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs);
  39
  40 /* 0 == registered but off, 1 == registered and on */
  41 static int nmi_enabled = 0;
  42
  43 #ifdef CONFIG_SMP
  44 static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
  45                                  void *data)
  46 {
  47         int cpu = (unsigned long)data;
  48         switch (action) {
  49         case CPU_DOWN_FAILED:
  50         case CPU_ONLINE:
  51                 smp_call_function_single(cpu, nmi_cpu_start, NULL, 0);
  52                 break;
  53         case CPU_DOWN_PREPARE:
  54                 smp_call_function_single(cpu, nmi_cpu_stop, NULL, 1);
  55                 break;
  56         }
  57         return NOTIFY_DONE;
  58 }
  59
  60 static struct notifier_block oprofile_cpu_nb = {
  61         .notifier_call = oprofile_cpu_notifier
  62 };
  63 #endif
  64
  65 #ifdef CONFIG_PM
  66
  67 static int nmi_suspend(struct sys_device *dev, pm_message_t state)
  68 {
  69         /* Only one CPU left, just stop that one */
  70         if (nmi_enabled == 1)
  71                 nmi_cpu_stop(NULL);
  72         return 0;
  73 }
  74
  75 static int nmi_resume(struct sys_device *dev)
  76 {
  77         if (nmi_enabled == 1)
  78                 nmi_cpu_start(NULL);
  79         return 0;
  80 }
  81
  82 static struct sysdev_class oprofile_sysclass = {
  83         .name           = "oprofile",
  84         .resume         = nmi_resume,
  85         .suspend        = nmi_suspend,
  86 };
  87
  88 static struct sys_device device_oprofile = {
  89         .id     = 0,
  90         .cls    = &oprofile_sysclass,
  91 };
  92
  93 static int __init init_sysfs(void)
  94 {
  95         int error;
  96
  97         error = sysdev_class_register(&oprofile_sysclass);
  98         if (!error)
  99                 error = sysdev_register(&device_oprofile);
 100         return error;
 101 }
 102
 103 static void exit_sysfs(void)
 104 {
 105         sysdev_unregister(&device_oprofile);
 106         sysdev_class_unregister(&oprofile_sysclass);
 107 }
 108
 109 #else
 110 #define init_sysfs() do { } while (0)
 111 #define exit_sysfs() do { } while (0)
 112 #endif /* CONFIG_PM */
 113
 114 static void nmi_cpu_switch(void *dummy)
 115 {
 116         int cpu = smp_processor_id();
 117         int si = per_cpu(switch_index, cpu);
 118         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 119
 120         nmi_cpu_stop(NULL);
 121         nmi_cpu_save_mpx_registers(msrs);
 122
 123         /* move to next set */
 124         si += model->num_hardware_counters;
 125         if ((si > model->num_counters) || (counter_config[si].count == 0))
 126                 per_cpu(switch_index, smp_processor_id()) = 0;
 127         else
 128                 per_cpu(switch_index, smp_processor_id()) = si;
 129
 130         nmi_cpu_restore_mpx_registers(msrs);
 131         model->setup_ctrs(msrs);
 132         nmi_cpu_start(NULL);
 133 }
 134
 135 /*
 136  * Quick check to see if multiplexing is necessary.
 137  * The check should be sufficient since counters are used
 138  * in ordre.
 139  */
 140 static int nmi_multiplex_on(void)
 141 {
 142         return counter_config[model->num_hardware_counters].count ? 0 : -EINVAL;
 143 }
 144
 145 static int nmi_switch_event(void)
 146 {
 147         if (nmi_multiplex_on() < 0)
 148                 return -EINVAL;
 149
 150         on_each_cpu(nmi_cpu_switch, NULL, 1);
 151
 152         return 0;
 153 }
 154
 155 static int profile_exceptions_notify(struct notifier_block *self,
 156                                      unsigned long val, void *data)
 157 {
 158         struct die_args *args = (struct die_args *)data;
 159         int ret = NOTIFY_DONE;
 160         int cpu = smp_processor_id();
 161
 162         switch (val) {
 163         case DIE_NMI:
 164                 if (model->check_ctrs(args->regs, &per_cpu(cpu_msrs, cpu)))
 165                         ret = NOTIFY_STOP;
 166                 break;
 167         default:
 168                 break;
 169         }
 170         return ret;
 171 }
 172
 173 static void nmi_cpu_save_registers(struct op_msrs *msrs)
 174 {
 175         unsigned int const nr_ctrs = model->num_counters;
 176         unsigned int const nr_ctrls = model->num_controls;
 177         struct op_msr *counters = msrs->counters;
 178         struct op_msr *controls = msrs->controls;
 179         unsigned int i;
 180
 181         for (i = 0; i < nr_ctrs; ++i) {
 182                 if (counters[i].addr) {
 183                         rdmsr(counters[i].addr,
 184                                 counters[i].saved.low,
 185                                 counters[i].saved.high);
 186                 }
 187         }
 188
 189         for (i = 0; i < nr_ctrls; ++i) {
 190                 if (controls[i].addr) {
 191                         rdmsr(controls[i].addr,
 192                                 controls[i].saved.low,
 193                                 controls[i].saved.high);
 194                 }
 195         }
 196 }
 197
 198 static void nmi_save_registers(void *dummy)
 199 {
 200         int cpu = smp_processor_id();
 201         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 202         nmi_cpu_save_registers(msrs);
 203 }
 204
 205 static void free_msrs(void)
 206 {
 207         int i;
 208         for_each_possible_cpu(i) {
 209                 kfree(per_cpu(cpu_msrs, i).counters);
 210                 per_cpu(cpu_msrs, i).counters = NULL;
 211                 kfree(per_cpu(cpu_msrs, i).controls);
 212                 per_cpu(cpu_msrs, i).controls = NULL;
 213         }
 214 }
 215
 216 static int allocate_msrs(void)
 217 {
 218         int i, success = 1;
 219         size_t controls_size = sizeof(struct op_msr) * model->num_controls;
 220         size_t counters_size = sizeof(struct op_msr) * model->num_counters;
 221
 222         for_each_possible_cpu(i) {
 223                 per_cpu(cpu_msrs, i).counters = kmalloc(counters_size,
 224                                                                 GFP_KERNEL);
 225                 if (!per_cpu(cpu_msrs, i).counters) {
 226                         success = 0;
 227                         break;
 228                 }
 229                 per_cpu(cpu_msrs, i).controls =
 230                                 kmalloc(controls_size, GFP_KERNEL);
 231                 if (!per_cpu(cpu_msrs, i).controls) {
 232                         success = 0;
 233                         break;
 234                 }
 235         }
 236
 237         if (!success)
 238                 free_msrs();
 239
 240         return success;
 241 }
 242
 243 static void nmi_cpu_setup(void *dummy)
 244 {
 245         int cpu = smp_processor_id();
 246         struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
 247         spin_lock(&oprofilefs_lock);
 248         model->setup_ctrs(msrs);
 249         spin_unlock(&oprofilefs_lock);
 250         per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC);
 251         apic_write(APIC_LVTPC, APIC_DM_NMI);
 252 }
 253
 254 static struct notifier_block profile_exceptions_nb = {
 255         .notifier_call = profile_exceptions_notify,
 256         .next = NULL,
 257         .priority = 0
 258 };
 259
 260 static int nmi_setup(void)
 261 {
 262         int err = 0;
 263         int cpu;
 264
 265         if (!allocate_msrs())
 266                 return -ENOMEM;
 267
 268         err = register_die_notifier(&profile_exceptions_nb);
 269         if (err) {
 270                 free_msrs();
 271                 return err;
 272         }
 273
 274         /*
 275          * We need to serialize save and setup for HT because the subset
 276          * of msrs are distinct for save and setup operations
 277          */
 278
 279         /* Assume saved/restored counters are the same on all CPUs */
 280         model->fill_in_addresses(&per_cpu(cpu_msrs, 0));
 281         for_each_possible_cpu(cpu) {
 282                 if (cpu != 0) {
 283                         memcpy(per_cpu(cpu_msrs, cpu).counters,
 284                                 per_cpu(cpu_msrs, 0).counters,
 285                                 sizeof(struct op_msr) * model->num_counters);
 286
 287                         memcpy(per_cpu(cpu_msrs, cpu).controls,
 288                                 per_cpu(cpu_msrs, 0).controls,
 289                                 sizeof(struct op_msr) * model->num_controls);
 290                 }
 291         }
 292         on_each_cpu(nmi_save_registers, NULL, 1);
 293         on_each_cpu(nmi_cpu_setup, NULL, 1);
 294         nmi_enabled = 1;
 295         return 0;
 296 }
 297
 298 static void nmi_cpu_save_mpx_registers(struct op_msrs *msrs)
 299 {
 300         unsigned int si = __get_cpu_var(switch_index);
 301         unsigned int const nr_ctrs = model->num_hardware_counters;
 302         struct op_msr *counters = &msrs->counters[si];
 303         unsigned int i;
 304
 305         for (i = 0; i < nr_ctrs; ++i) {
 306                 int offset = i + si;
 307                 if (counters[offset].addr) {
 308                         rdmsr(counters[offset].addr,
 309                                 counters[offset].multiplex.low,
 310                                 counters[offset].multiplex.high);
 311                 }
 312         }
 313 }
 314
 315 static void nmi_cpu_restore_mpx_registers(struct op_msrs *msrs)
 316 {
 317         unsigned int si = __get_cpu_var(switch_index);
 318         unsigned int const nr_ctrs = model->num_hardware_counters;
 319         struct op_msr *counters = &msrs->counters[si];
 320         unsigned int i;
 321
 322         for (i = 0; i < nr_ctrs; ++i) {
 323                 int offset = i + si;
 324                 if (counters[offset].addr) {
 325                         wrmsr(counters[offset].addr,
 326                                 counters[offset].multiplex.low,
 327                                 counters[offset].multiplex.high);
 328                 }
 329         }
 330 }
 331
 332 static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 333 {
 334         unsigned int const nr_ctrs = model->num_counters;
 335         unsigned int const nr_ctrls = model->num_controls;
 336         struct op_msr *counters = msrs->counters;
 337         struct op_msr *controls = msrs->controls;
 338         unsigned int i;
 339
 340         for (i = 0; i < nr_ctrls; ++i) {
 341                 if (controls[i].addr) {
 342                         wrmsr(controls[i].addr,
 343                                 controls[i].saved.low,
 344                                 controls[i].saved.high);
 345                 }
 346         }
 347
 348         for (i = 0; i < nr_ctrs; ++i) {
 349                 if (counters[i].addr) {
 350                         wrmsr(counters[i].addr,
 351                                 counters[i].saved.low,
 352                                 counters[i].saved.high);
 353                 }
 354         }
 355 }
 356
 357 static void nmi_cpu_shutdown(void *dummy)
 358 {
 359         unsigned int v;
 360         int cpu = smp_processor_id();
 361         struct op_msrs *msrs = &__get_cpu_var(cpu_msrs);
 362
 363         /* restoring APIC_LVTPC can trigger an apic error because the delivery
 364          * mode and vector nr combination can be illegal. That's by design: on
 365          * power on apic lvt contain a zero vector nr which are legal only for
 366          * NMI delivery mode. So inhibit apic err before restoring lvtpc
 367          */
 368         v = apic_read(APIC_LVTERR);
 369         apic_write(APIC_LVTERR, v | APIC_LVT_MASKED);
 370         apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu));
 371         apic_write(APIC_LVTERR, v);
 372         nmi_cpu_restore_registers(msrs);
 373         __get_cpu_var(switch_index) = 0;
 374 }
 375
 376 static void nmi_shutdown(void)
 377 {
 378         struct op_msrs *msrs;
 379
 380         nmi_enabled = 0;
 381         on_each_cpu(nmi_cpu_shutdown, NULL, 1);
 382         unregister_die_notifier(&profile_exceptions_nb);
 383         msrs = &get_cpu_var(cpu_msrs);
 384         model->shutdown(msrs);
 385         free_msrs();
 386         put_cpu_var(cpu_msrs);
 387 }
 388
 389 static void nmi_cpu_start(void *dummy)
 390 {
 391         struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
 392         model->start(msrs);
 393 }
 394
 395 static int nmi_start(void)
 396 {
 397         on_each_cpu(nmi_cpu_start, NULL, 1);
 398         return 0;
 399 }
 400
 401 static void nmi_cpu_stop(void *dummy)
 402 {
 403         struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs);
 404         model->stop(msrs);
 405 }
 406
 407 static void nmi_stop(void)
 408 {
 409         on_each_cpu(nmi_cpu_stop, NULL, 1);
 410 }
 411
 412 struct op_counter_config counter_config[OP_MAX_COUNTER];
 413
 414 static int nmi_create_files(struct super_block *sb, struct dentry *root)
 415 {
 416         unsigned int i;
 417
 418         for (i = 0; i < model->num_counters; ++i) {
 419                 struct dentry *dir;
 420                 char buf[4];
 421
 422                 /* quick little hack to _not_ expose a counter if it is not
 423                  * available for use.  This should protect userspace app.
 424                  * NOTE:  assumes 1:1 mapping here (that counters are organized
 425                  *        sequentially in their struct assignment).
 426                  */
 427                 if (unlikely(!avail_to_resrv_perfctr_nmi_bit(i)))
 428                         continue;
 429
 430                 snprintf(buf,  sizeof(buf), "%d", i);
 431                 dir = oprofilefs_mkdir(sb, root, buf);
 432                 oprofilefs_create_ulong(sb, dir, "enabled", &counter_config[i].enabled);
 433                 oprofilefs_create_ulong(sb, dir, "event", &counter_config[i].event);
 434                 oprofilefs_create_ulong(sb, dir, "count", &counter_config[i].count);
 435                 oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask);
 436                 oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel);
 437                 oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user);
 438                 counter_config[i].save_count_low = 0;
 439         }
 440
 441         return 0;
 442 }
 443
 444 static int p4force;
 445 module_param(p4force, int, 0);
 446
 447 static int __init p4_init(char **cpu_type)
 448 {
 449         __u8 cpu_model = boot_cpu_data.x86_model;
 450
 451         if (!p4force && (cpu_model > 6 || cpu_model == 5))
 452                 return 0;
 453
 454 #ifndef CONFIG_SMP
 455         *cpu_type = "i386/p4";
 456         model = &op_p4_spec;
 457         return 1;
 458 #else
 459         switch (smp_num_siblings) {
 460         case 1:
 461                 *cpu_type = "i386/p4";
 462                 model = &op_p4_spec;
 463                 return 1;
 464
 465         case 2:
 466                 *cpu_type = "i386/p4-ht";
 467                 model = &op_p4_ht2_spec;
 468                 return 1;
 469         }
 470 #endif
 471
 472         printk(KERN_INFO "oprofile: P4 HyperThreading detected with > 2 threads\n");
 473         printk(KERN_INFO "oprofile: Reverting to timer mode.\n");
 474         return 0;
 475 }
 476
 477 static int __init ppro_init(char **cpu_type)
 478 {
 479         __u8 cpu_model = boot_cpu_data.x86_model;
 480
 481         switch (cpu_model) {
 482         case 0 ... 2:
 483                 *cpu_type = "i386/ppro";
 484                 break;
 485         case 3 ... 5:
 486                 *cpu_type = "i386/pii";
 487                 break;
 488         case 6 ... 8:
 489                 *cpu_type = "i386/piii";
 490                 break;
 491         case 9:
 492                 *cpu_type = "i386/p6_mobile";
 493                 break;
 494         case 10 ... 13:
 495                 *cpu_type = "i386/p6";
 496                 break;
 497         case 14:
 498                 *cpu_type = "i386/core";
 499                 break;
 500         case 15: case 23:
 501                 *cpu_type = "i386/core_2";
 502                 break;
 503         case 26:
 504                 *cpu_type = "i386/core_2";
 505                 break;
 506         default:
 507                 /* Unknown */
 508                 return 0;
 509         }
 510
 511         model = &op_ppro_spec;
 512         return 1;
 513 }
 514
 515 /* in order to get sysfs right */
 516 static int using_nmi;
 517
 518 int __init op_nmi_init(struct oprofile_operations *ops)
 519 {
 520         __u8 vendor = boot_cpu_data.x86_vendor;
 521         __u8 family = boot_cpu_data.x86;
 522         char *cpu_type;
 523         int ret = 0;
 524
 525         if (!cpu_has_apic)
 526                 return -ENODEV;
 527
 528         switch (vendor) {
 529         case X86_VENDOR_AMD:
 530                 /* Needs to be at least an Athlon (or hammer in 32bit mode) */
 531
 532                 switch (family) {
 533                 default:
 534                         return -ENODEV;
 535                 case 6:
 536                         model = &op_amd_spec;
 537                         cpu_type = "i386/athlon";
 538                         break;
 539                 case 0xf:
 540                         model = &op_amd_spec;
 541                         /* Actually it could be i386/hammer too, but give
 542                          user space an consistent name. */
 543                         cpu_type = "x86-64/hammer";
 544                         break;
 545                 case 0x10:
 546                         model = &op_amd_spec;
 547                         cpu_type = "x86-64/family10";
 548                         break;
 549                 case 0x11:
 550                         model = &op_amd_spec;
 551                         cpu_type = "x86-64/family11h";
 552                         break;
 553                 }
 554                 break;
 555
 556         case X86_VENDOR_INTEL:
 557                 switch (family) {
 558                         /* Pentium IV */
 559                 case 0xf:
 560                         if (!p4_init(&cpu_type))
 561                                 return -ENODEV;
 562                         break;
 563
 564                         /* A P6-class processor */
 565                 case 6:
 566                         if (!ppro_init(&cpu_type))
 567                                 return -ENODEV;
 568                         break;
 569
 570                 default:
 571                         return -ENODEV;
 572                 }
 573                 break;
 574
 575         default:
 576                 return -ENODEV;
 577         }
 578
 579 #ifdef CONFIG_SMP
 580         register_cpu_notifier(&oprofile_cpu_nb);
 581 #endif
 582         /* default values, can be overwritten by model */
 583         __raw_get_cpu_var(switch_index) = 0;
 584         ops->create_files = nmi_create_files;
 585         ops->setup = nmi_setup;
 586         ops->shutdown = nmi_shutdown;
 587         ops->start = nmi_start;
 588         ops->stop = nmi_stop;
 589         ops->cpu_type = cpu_type;
 590         ops->switch_events = nmi_switch_event;
 591
 592         if (model->init)
 593                 ret = model->init(ops);
 594         if (ret)
 595                 return ret;
 596
 597         init_sysfs();
 598         using_nmi = 1;
 599         printk(KERN_INFO "oprofile: using NMI interrupt.\n");
 600         return 0;
 601 }
 602
 603 void op_nmi_exit(void)
 604 {
 605         if (using_nmi) {
 606                 exit_sysfs();
 607 #ifdef CONFIG_SMP
 608                 unregister_cpu_notifier(&oprofile_cpu_nb);
 609 #endif
 610         if (model->exit)
 611                 model->exit();
 612         }
 613 }