arch/x86/oprofile/op_model_amd.c

   1 /*
   2  * @file op_model_amd.c
   3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
   4  *
   5  * @remark Copyright 2002-2009 OProfile authors
   6  * @remark Read the file COPYING
   7  *
   8  * @author John Levon
   9  * @author Philippe Elie
  10  * @author Graydon Hoare
  11  * @author Robert Richter <robert.richter@amd.com>
  12  * @author Barry Kasindorf <barry.kasindorf@amd.com>
  13  * @author Jason Yeh <jason.yeh@amd.com>
  14  * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
  15  */
  16
  17 #include <linux/oprofile.h>
  18 #include <linux/device.h>
  19 #include <linux/pci.h>
  20 #include <linux/percpu.h>
  21
  22 #include <asm/ptrace.h>
  23 #include <asm/msr.h>
  24 #include <asm/nmi.h>
  25 #include <asm/apic.h>
  26 #include <asm/processor.h>
  27 #include <asm/cpufeature.h>
  28
  29 #include "op_x86_model.h"
  30 #include "op_counter.h"
  31
  32 #define NUM_COUNTERS 4
  33 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
  34 #define NUM_VIRT_COUNTERS 32
  35 #else
  36 #define NUM_VIRT_COUNTERS NUM_COUNTERS
  37 #endif
  38
  39 #define OP_EVENT_MASK                   0x0FFF
  40 #define OP_CTR_OVERFLOW                 (1ULL<<31)
  41
  42 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
  43
  44 static unsigned long reset_value[NUM_VIRT_COUNTERS];
  45
  46 #define IBS_FETCH_SIZE                  6
  47 #define IBS_OP_SIZE                     12
  48
  49 static u32 ibs_caps;
  50
  51 struct op_ibs_config {
  52         unsigned long op_enabled;
  53         unsigned long fetch_enabled;
  54         unsigned long max_cnt_fetch;
  55         unsigned long max_cnt_op;
  56         unsigned long rand_en;
  57         unsigned long dispatched_ops;
  58 };
  59
  60 static struct op_ibs_config ibs_config;
  61 static u64 ibs_op_ctl;
  62
  63 /*
  64  * IBS cpuid feature detection
  65  */
  66
  67 #define IBS_CPUID_FEATURES      0x8000001b
  68
  69 /*
  70  * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
  71  * bit 0 is used to indicate the existence of IBS.
  72  */
  73 #define IBS_CAPS_AVAIL                  (1U<<0)
  74 #define IBS_CAPS_FETCHSAM               (1U<<1)
  75 #define IBS_CAPS_OPSAM                  (1U<<2)
  76 #define IBS_CAPS_RDWROPCNT              (1U<<3)
  77 #define IBS_CAPS_OPCNT                  (1U<<4)
  78
  79 #define IBS_CAPS_DEFAULT                (IBS_CAPS_AVAIL         \
  80                                          | IBS_CAPS_FETCHSAM    \
  81                                          | IBS_CAPS_OPSAM)
  82
  83 /*
  84  * IBS APIC setup
  85  */
  86 #define IBSCTL                          0x1cc
  87 #define IBSCTL_LVT_OFFSET_VALID         (1ULL<<8)
  88 #define IBSCTL_LVT_OFFSET_MASK          0x0F
  89
  90 /*
  91  * IBS randomization macros
  92  */
  93 #define IBS_RANDOM_BITS                 12
  94 #define IBS_RANDOM_MASK                 ((1ULL << IBS_RANDOM_BITS) - 1)
  95 #define IBS_RANDOM_MAXCNT_OFFSET        (1ULL << (IBS_RANDOM_BITS - 5))
  96
  97 static u32 get_ibs_caps(void)
  98 {
  99         u32 ibs_caps;
 100         unsigned int max_level;
 101
 102         if (!boot_cpu_has(X86_FEATURE_IBS))
 103                 return 0;
 104
 105         /* check IBS cpuid feature flags */
 106         max_level = cpuid_eax(0x80000000);
 107         if (max_level < IBS_CPUID_FEATURES)
 108                 return IBS_CAPS_DEFAULT;
 109
 110         ibs_caps = cpuid_eax(IBS_CPUID_FEATURES);
 111         if (!(ibs_caps & IBS_CAPS_AVAIL))
 112                 /* cpuid flags not valid */
 113                 return IBS_CAPS_DEFAULT;
 114
 115         return ibs_caps;
 116 }
 117
 118 /*
 119  * 16-bit Linear Feedback Shift Register (LFSR)
 120  *
 121  *                       16   14   13    11
 122  * Feedback polynomial = X  + X  + X  +  X  + 1
 123  */
 124 static unsigned int lfsr_random(void)
 125 {
 126         static unsigned int lfsr_value = 0xF00D;
 127         unsigned int bit;
 128
 129         /* Compute next bit to shift in */
 130         bit = ((lfsr_value >> 0) ^
 131                (lfsr_value >> 2) ^
 132                (lfsr_value >> 3) ^
 133                (lfsr_value >> 5)) & 0x0001;
 134
 135         /* Advance to next register value */
 136         lfsr_value = (lfsr_value >> 1) | (bit << 15);
 137
 138         return lfsr_value;
 139 }
 140
 141 /*
 142  * IBS software randomization
 143  *
 144  * The IBS periodic op counter is randomized in software. The lower 12
 145  * bits of the 20 bit counter are randomized. IbsOpCurCnt is
 146  * initialized with a 12 bit random value.
 147  */
 148 static inline u64 op_amd_randomize_ibs_op(u64 val)
 149 {
 150         unsigned int random = lfsr_random();
 151
 152         if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
 153                 /*
 154                  * Work around if the hw can not write to IbsOpCurCnt
 155                  *
 156                  * Randomize the lower 8 bits of the 16 bit
 157                  * IbsOpMaxCnt [15:0] value in the range of -128 to
 158                  * +127 by adding/subtracting an offset to the
 159                  * maximum count (IbsOpMaxCnt).
 160                  *
 161                  * To avoid over or underflows and protect upper bits
 162                  * starting at bit 16, the initial value for
 163                  * IbsOpMaxCnt must fit in the range from 0x0081 to
 164                  * 0xff80.
 165                  */
 166                 val += (s8)(random >> 4);
 167         else
 168                 val |= (u64)(random & IBS_RANDOM_MASK) << 32;
 169
 170         return val;
 171 }
 172
 173 static inline void
 174 op_amd_handle_ibs(struct pt_regs * const regs,
 175                   struct op_msrs const * const msrs)
 176 {
 177         u64 val, ctl;
 178         struct op_entry entry;
 179
 180         if (!ibs_caps)
 181                 return;
 182
 183         if (ibs_config.fetch_enabled) {
 184                 rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
 185                 if (ctl & IBS_FETCH_VAL) {
 186                         rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
 187                         oprofile_write_reserve(&entry, regs, val,
 188                                                IBS_FETCH_CODE, IBS_FETCH_SIZE);
 189                         oprofile_add_data64(&entry, val);
 190                         oprofile_add_data64(&entry, ctl);
 191                         rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
 192                         oprofile_add_data64(&entry, val);
 193                         oprofile_write_commit(&entry);
 194
 195                         /* reenable the IRQ */
 196                         ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
 197                         ctl |= IBS_FETCH_ENABLE;
 198                         wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
 199                 }
 200         }
 201
 202         if (ibs_config.op_enabled) {
 203                 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
 204                 if (ctl & IBS_OP_VAL) {
 205                         rdmsrl(MSR_AMD64_IBSOPRIP, val);
 206                         oprofile_write_reserve(&entry, regs, val,
 207                                                IBS_OP_CODE, IBS_OP_SIZE);
 208                         oprofile_add_data64(&entry, val);
 209                         rdmsrl(MSR_AMD64_IBSOPDATA, val);
 210                         oprofile_add_data64(&entry, val);
 211                         rdmsrl(MSR_AMD64_IBSOPDATA2, val);
 212                         oprofile_add_data64(&entry, val);
 213                         rdmsrl(MSR_AMD64_IBSOPDATA3, val);
 214                         oprofile_add_data64(&entry, val);
 215                         rdmsrl(MSR_AMD64_IBSDCLINAD, val);
 216                         oprofile_add_data64(&entry, val);
 217                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
 218                         oprofile_add_data64(&entry, val);
 219                         oprofile_write_commit(&entry);
 220
 221                         /* reenable the IRQ */
 222                         ctl = op_amd_randomize_ibs_op(ibs_op_ctl);
 223                         wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
 224                 }
 225         }
 226 }
 227
 228 static inline void op_amd_start_ibs(void)
 229 {
 230         u64 val;
 231
 232         if (!ibs_caps)
 233                 return;
 234
 235         if (ibs_config.fetch_enabled) {
 236                 val = (ibs_config.max_cnt_fetch >> 4) & IBS_FETCH_MAX_CNT;
 237                 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
 238                 val |= IBS_FETCH_ENABLE;
 239                 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
 240         }
 241
 242         if (ibs_config.op_enabled) {
 243                 ibs_op_ctl = ibs_config.max_cnt_op >> 4;
 244                 if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
 245                         /*
 246                          * IbsOpCurCnt not supported.  See
 247                          * op_amd_randomize_ibs_op() for details.
 248                          */
 249                         ibs_op_ctl = clamp(ibs_op_ctl, 0x0081ULL, 0xFF80ULL);
 250                 } else {
 251                         /*
 252                          * The start value is randomized with a
 253                          * positive offset, we need to compensate it
 254                          * with the half of the randomized range. Also
 255                          * avoid underflows.
 256                          */
 257                         ibs_op_ctl = min(ibs_op_ctl + IBS_RANDOM_MAXCNT_OFFSET,
 258                                          IBS_OP_MAX_CNT);
 259                 }
 260                 if (ibs_caps & IBS_CAPS_OPCNT && ibs_config.dispatched_ops)
 261                         ibs_op_ctl |= IBS_OP_CNT_CTL;
 262                 ibs_op_ctl |= IBS_OP_ENABLE;
 263                 val = op_amd_randomize_ibs_op(ibs_op_ctl);
 264                 wrmsrl(MSR_AMD64_IBSOPCTL, val);
 265         }
 266 }
 267
 268 static void op_amd_stop_ibs(void)
 269 {
 270         if (!ibs_caps)
 271                 return;
 272
 273         if (ibs_config.fetch_enabled)
 274                 /* clear max count and enable */
 275                 wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
 276
 277         if (ibs_config.op_enabled)
 278                 /* clear max count and enable */
 279                 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
 280 }
 281
 282 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 283
 284 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
 285                                struct op_msrs const * const msrs)
 286 {
 287         u64 val;
 288         int i;
 289
 290         /* enable active counters */
 291         for (i = 0; i < NUM_COUNTERS; ++i) {
 292                 int virt = op_x86_phys_to_virt(i);
 293                 if (!reset_value[virt])
 294                         continue;
 295                 rdmsrl(msrs->controls[i].addr, val);
 296                 val &= model->reserved;
 297                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
 298                 wrmsrl(msrs->controls[i].addr, val);
 299         }
 300 }
 301
 302 #endif
 303
 304 /* functions for op_amd_spec */
 305
 306 static void op_amd_shutdown(struct op_msrs const * const msrs)
 307 {
 308         int i;
 309
 310         for (i = 0; i < NUM_COUNTERS; ++i) {
 311                 if (!msrs->counters[i].addr)
 312                         continue;
 313                 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 314                 release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
 315         }
 316 }
 317
 318 static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
 319 {
 320         int i;
 321
 322         for (i = 0; i < NUM_COUNTERS; i++) {
 323                 if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
 324                         goto fail;
 325                 if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
 326                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 327                         goto fail;
 328                 }
 329                 /* both registers must be reserved */
 330                 msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
 331                 msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
 332                 continue;
 333         fail:
 334                 if (!counter_config[i].enabled)
 335                         continue;
 336                 op_x86_warn_reserved(i);
 337                 op_amd_shutdown(msrs);
 338                 return -EBUSY;
 339         }
 340
 341         return 0;
 342 }
 343
 344 static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
 345                               struct op_msrs const * const msrs)
 346 {
 347         u64 val;
 348         int i;
 349
 350         /* setup reset_value */
 351         for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
 352                 if (counter_config[i].enabled
 353                     && msrs->counters[op_x86_virt_to_phys(i)].addr)
 354                         reset_value[i] = counter_config[i].count;
 355                 else
 356                         reset_value[i] = 0;
 357         }
 358
 359         /* clear all counters */
 360         for (i = 0; i < NUM_COUNTERS; ++i) {
 361                 if (!msrs->controls[i].addr)
 362                         continue;
 363                 rdmsrl(msrs->controls[i].addr, val);
 364                 if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
 365                         op_x86_warn_in_use(i);
 366                 val &= model->reserved;
 367                 wrmsrl(msrs->controls[i].addr, val);
 368                 /*
 369                  * avoid a false detection of ctr overflows in NMI
 370                  * handler
 371                  */
 372                 wrmsrl(msrs->counters[i].addr, -1LL);
 373         }
 374
 375         /* enable active counters */
 376         for (i = 0; i < NUM_COUNTERS; ++i) {
 377                 int virt = op_x86_phys_to_virt(i);
 378                 if (!reset_value[virt])
 379                         continue;
 380
 381                 /* setup counter registers */
 382                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
 383
 384                 /* setup control registers */
 385                 rdmsrl(msrs->controls[i].addr, val);
 386                 val &= model->reserved;
 387                 val |= op_x86_get_ctrl(model, &counter_config[virt]);
 388                 wrmsrl(msrs->controls[i].addr, val);
 389         }
 390
 391         if (ibs_caps)
 392                 setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
 393 }
 394
 395 static void op_amd_cpu_shutdown(void)
 396 {
 397         if (ibs_caps)
 398                 setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 399 }
 400
 401 static int op_amd_check_ctrs(struct pt_regs * const regs,
 402                              struct op_msrs const * const msrs)
 403 {
 404         u64 val;
 405         int i;
 406
 407         for (i = 0; i < NUM_COUNTERS; ++i) {
 408                 int virt = op_x86_phys_to_virt(i);
 409                 if (!reset_value[virt])
 410                         continue;
 411                 rdmsrl(msrs->counters[i].addr, val);
 412                 /* bit is clear if overflowed: */
 413                 if (val & OP_CTR_OVERFLOW)
 414                         continue;
 415                 oprofile_add_sample(regs, virt);
 416                 wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
 417         }
 418
 419         op_amd_handle_ibs(regs, msrs);
 420
 421         /* See op_model_ppro.c */
 422         return 1;
 423 }
 424
 425 static void op_amd_start(struct op_msrs const * const msrs)
 426 {
 427         u64 val;
 428         int i;
 429
 430         for (i = 0; i < NUM_COUNTERS; ++i) {
 431                 if (!reset_value[op_x86_phys_to_virt(i)])
 432                         continue;
 433                 rdmsrl(msrs->controls[i].addr, val);
 434                 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
 435                 wrmsrl(msrs->controls[i].addr, val);
 436         }
 437
 438         op_amd_start_ibs();
 439 }
 440
 441 static void op_amd_stop(struct op_msrs const * const msrs)
 442 {
 443         u64 val;
 444         int i;
 445
 446         /*
 447          * Subtle: stop on all counters to avoid race with setting our
 448          * pm callback
 449          */
 450         for (i = 0; i < NUM_COUNTERS; ++i) {
 451                 if (!reset_value[op_x86_phys_to_virt(i)])
 452                         continue;
 453                 rdmsrl(msrs->controls[i].addr, val);
 454                 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
 455                 wrmsrl(msrs->controls[i].addr, val);
 456         }
 457
 458         op_amd_stop_ibs();
 459 }
 460
 461 static int __init_ibs_nmi(void)
 462 {
 463 #define IBSCTL_LVTOFFSETVAL             (1 << 8)
 464 #define IBSCTL                          0x1cc
 465         struct pci_dev *cpu_cfg;
 466         int nodes;
 467         u32 value = 0;
 468         u8 ibs_eilvt_off;
 469
 470         ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 471
 472         nodes = 0;
 473         cpu_cfg = NULL;
 474         do {
 475                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
 476                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
 477                                          cpu_cfg);
 478                 if (!cpu_cfg)
 479                         break;
 480                 ++nodes;
 481                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
 482                                        | IBSCTL_LVTOFFSETVAL);
 483                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
 484                 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
 485                         pci_dev_put(cpu_cfg);
 486                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
 487                                 "IBSCTL = 0x%08x", value);
 488                         return 1;
 489                 }
 490         } while (1);
 491
 492         if (!nodes) {
 493                 printk(KERN_DEBUG "No CPU node configured for IBS");
 494                 return 1;
 495         }
 496
 497         return 0;
 498 }
 499
 500 /* initialize the APIC for the IBS interrupts if available */
 501 static void init_ibs(void)
 502 {
 503         ibs_caps = get_ibs_caps();
 504
 505         if (!ibs_caps)
 506                 return;
 507
 508         if (__init_ibs_nmi()) {
 509                 ibs_caps = 0;
 510                 return;
 511         }
 512
 513         printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n",
 514                (unsigned)ibs_caps);
 515 }
 516
 517 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
 518
 519 static int setup_ibs_files(struct super_block *sb, struct dentry *root)
 520 {
 521         struct dentry *dir;
 522         int ret = 0;
 523
 524         /* architecture specific files */
 525         if (create_arch_files)
 526                 ret = create_arch_files(sb, root);
 527
 528         if (ret)
 529                 return ret;
 530
 531         if (!ibs_caps)
 532                 return ret;
 533
 534         /* model specific files */
 535
 536         /* setup some reasonable defaults */
 537         ibs_config.max_cnt_fetch = 250000;
 538         ibs_config.fetch_enabled = 0;
 539         ibs_config.max_cnt_op = 250000;
 540         ibs_config.op_enabled = 0;
 541         ibs_config.dispatched_ops = 0;
 542
 543         if (ibs_caps & IBS_CAPS_FETCHSAM) {
 544                 dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
 545                 oprofilefs_create_ulong(sb, dir, "enable",
 546                                         &ibs_config.fetch_enabled);
 547                 oprofilefs_create_ulong(sb, dir, "max_count",
 548                                         &ibs_config.max_cnt_fetch);
 549                 oprofilefs_create_ulong(sb, dir, "rand_enable",
 550                                         &ibs_config.rand_en);
 551         }
 552
 553         if (ibs_caps & IBS_CAPS_OPSAM) {
 554                 dir = oprofilefs_mkdir(sb, root, "ibs_op");
 555                 oprofilefs_create_ulong(sb, dir, "enable",
 556                                         &ibs_config.op_enabled);
 557                 oprofilefs_create_ulong(sb, dir, "max_count",
 558                                         &ibs_config.max_cnt_op);
 559                 if (ibs_caps & IBS_CAPS_OPCNT)
 560                         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
 561                                                 &ibs_config.dispatched_ops);
 562         }
 563
 564         return 0;
 565 }
 566
 567 static int op_amd_init(struct oprofile_operations *ops)
 568 {
 569         init_ibs();
 570         create_arch_files = ops->create_files;
 571         ops->create_files = setup_ibs_files;
 572         return 0;
 573 }
 574
 575 struct op_x86_model_spec op_amd_spec = {
 576         .num_counters           = NUM_COUNTERS,
 577         .num_controls           = NUM_COUNTERS,
 578         .num_virt_counters      = NUM_VIRT_COUNTERS,
 579         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
 580         .event_mask             = OP_EVENT_MASK,
 581         .init                   = op_amd_init,
 582         .fill_in_addresses      = &op_amd_fill_in_addresses,
 583         .setup_ctrs             = &op_amd_setup_ctrs,
 584         .cpu_down               = &op_amd_cpu_shutdown,
 585         .check_ctrs             = &op_amd_check_ctrs,
 586         .start                  = &op_amd_start,
 587         .stop                   = &op_amd_stop,
 588         .shutdown               = &op_amd_shutdown,
 589 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 590         .switch_ctrl            = &op_mux_switch_ctrl,
 591 #endif
 592 };