arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *
  10  *  For licencing details see kernel-base/COPYING
  11  */
  12
  13 #include <linux/perf_counter.h>
  14 #include <linux/capability.h>
  15 #include <linux/notifier.h>
  16 #include <linux/hardirq.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/module.h>
  19 #include <linux/kdebug.h>
  20 #include <linux/sched.h>
  21 #include <linux/uaccess.h>
  22
  23 #include <asm/apic.h>
  24 #include <asm/stacktrace.h>
  25 #include <asm/nmi.h>
  26
  27 static u64 perf_counter_mask __read_mostly;
  28
  29 struct cpu_hw_counters {
  30         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  31         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  32         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  33         unsigned long           interrupts;
  34         int                     enabled;
  35 };
  36
  37 /*
  38  * struct x86_pmu - generic x86 pmu
  39  */
  40 struct x86_pmu {
  41         const char      *name;
  42         int             version;
  43         int             (*handle_irq)(struct pt_regs *, int);
  44         void            (*disable_all)(void);
  45         void            (*enable_all)(void);
  46         void            (*enable)(struct hw_perf_counter *, int);
  47         void            (*disable)(struct hw_perf_counter *, int);
  48         unsigned        eventsel;
  49         unsigned        perfctr;
  50         u64             (*event_map)(int);
  51         u64             (*raw_event)(u64);
  52         int             max_events;
  53         int             num_counters;
  54         int             num_counters_fixed;
  55         int             counter_bits;
  56         u64             counter_mask;
  57         u64             max_period;
  58         u64             intel_ctrl;
  59 };
  60
  61 static struct x86_pmu x86_pmu __read_mostly;
  62
  63 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  64         .enabled = 1,
  65 };
  66
  67 /*
  68  * Intel PerfMon v3. Used on Core2 and later.
  69  */
  70 static const u64 intel_perfmon_event_map[] =
  71 {
  72   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  73   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  74   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  75   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  76   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  77   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  78   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  79 };
  80
  81 static u64 intel_pmu_event_map(int event)
  82 {
  83         return intel_perfmon_event_map[event];
  84 }
  85
  86 static u64 intel_pmu_raw_event(u64 event)
  87 {
  88 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
  89 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
  90 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
  91 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
  92 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
  93
  94 #define CORE_EVNTSEL_MASK               \
  95         (CORE_EVNTSEL_EVENT_MASK |      \
  96          CORE_EVNTSEL_UNIT_MASK  |      \
  97          CORE_EVNTSEL_EDGE_MASK  |      \
  98          CORE_EVNTSEL_INV_MASK  |       \
  99          CORE_EVNTSEL_COUNTER_MASK)
 100
 101         return event & CORE_EVNTSEL_MASK;
 102 }
 103
 104 /*
 105  * AMD Performance Monitor K7 and later.
 106  */
 107 static const u64 amd_perfmon_event_map[] =
 108 {
 109   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
 110   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
 111   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
 112   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
 113   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
 114   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
 115 };
 116
 117 static u64 amd_pmu_event_map(int event)
 118 {
 119         return amd_perfmon_event_map[event];
 120 }
 121
 122 static u64 amd_pmu_raw_event(u64 event)
 123 {
 124 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 125 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 126 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 127 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 128 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 129
 130 #define K7_EVNTSEL_MASK                 \
 131         (K7_EVNTSEL_EVENT_MASK |        \
 132          K7_EVNTSEL_UNIT_MASK  |        \
 133          K7_EVNTSEL_EDGE_MASK  |        \
 134          K7_EVNTSEL_INV_MASK   |        \
 135          K7_EVNTSEL_COUNTER_MASK)
 136
 137         return event & K7_EVNTSEL_MASK;
 138 }
 139
 140 /*
 141  * Propagate counter elapsed time into the generic counter.
 142  * Can only be executed on the CPU where the counter is active.
 143  * Returns the delta events processed.
 144  */
 145 static u64
 146 x86_perf_counter_update(struct perf_counter *counter,
 147                         struct hw_perf_counter *hwc, int idx)
 148 {
 149         int shift = 64 - x86_pmu.counter_bits;
 150         u64 prev_raw_count, new_raw_count;
 151         s64 delta;
 152
 153         /*
 154          * Careful: an NMI might modify the previous counter value.
 155          *
 156          * Our tactic to handle this is to first atomically read and
 157          * exchange a new raw count - then add that new-prev delta
 158          * count to the generic counter atomically:
 159          */
 160 again:
 161         prev_raw_count = atomic64_read(&hwc->prev_count);
 162         rdmsrl(hwc->counter_base + idx, new_raw_count);
 163
 164         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 165                                         new_raw_count) != prev_raw_count)
 166                 goto again;
 167
 168         /*
 169          * Now we have the new raw value and have updated the prev
 170          * timestamp already. We can now calculate the elapsed delta
 171          * (counter-)time and add that to the generic counter.
 172          *
 173          * Careful, not all hw sign-extends above the physical width
 174          * of the count.
 175          */
 176         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 177         delta >>= shift;
 178
 179         atomic64_add(delta, &counter->count);
 180         atomic64_sub(delta, &hwc->period_left);
 181
 182         return new_raw_count;
 183 }
 184
 185 static atomic_t active_counters;
 186 static DEFINE_MUTEX(pmc_reserve_mutex);
 187
 188 static bool reserve_pmc_hardware(void)
 189 {
 190         int i;
 191
 192         if (nmi_watchdog == NMI_LOCAL_APIC)
 193                 disable_lapic_nmi_watchdog();
 194
 195         for (i = 0; i < x86_pmu.num_counters; i++) {
 196                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 197                         goto perfctr_fail;
 198         }
 199
 200         for (i = 0; i < x86_pmu.num_counters; i++) {
 201                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 202                         goto eventsel_fail;
 203         }
 204
 205         return true;
 206
 207 eventsel_fail:
 208         for (i--; i >= 0; i--)
 209                 release_evntsel_nmi(x86_pmu.eventsel + i);
 210
 211         i = x86_pmu.num_counters;
 212
 213 perfctr_fail:
 214         for (i--; i >= 0; i--)
 215                 release_perfctr_nmi(x86_pmu.perfctr + i);
 216
 217         if (nmi_watchdog == NMI_LOCAL_APIC)
 218                 enable_lapic_nmi_watchdog();
 219
 220         return false;
 221 }
 222
 223 static void release_pmc_hardware(void)
 224 {
 225         int i;
 226
 227         for (i = 0; i < x86_pmu.num_counters; i++) {
 228                 release_perfctr_nmi(x86_pmu.perfctr + i);
 229                 release_evntsel_nmi(x86_pmu.eventsel + i);
 230         }
 231
 232         if (nmi_watchdog == NMI_LOCAL_APIC)
 233                 enable_lapic_nmi_watchdog();
 234 }
 235
 236 static void hw_perf_counter_destroy(struct perf_counter *counter)
 237 {
 238         if (atomic_dec_and_mutex_lock(&active_counters, &pmc_reserve_mutex)) {
 239                 release_pmc_hardware();
 240                 mutex_unlock(&pmc_reserve_mutex);
 241         }
 242 }
 243
 244 static inline int x86_pmu_initialized(void)
 245 {
 246         return x86_pmu.handle_irq != NULL;
 247 }
 248
 249 /*
 250  * Setup the hardware configuration for a given hw_event_type
 251  */
 252 static int __hw_perf_counter_init(struct perf_counter *counter)
 253 {
 254         struct perf_counter_hw_event *hw_event = &counter->hw_event;
 255         struct hw_perf_counter *hwc = &counter->hw;
 256         int err;
 257
 258         if (!x86_pmu_initialized())
 259                 return -ENODEV;
 260
 261         err = 0;
 262         if (!atomic_inc_not_zero(&active_counters)) {
 263                 mutex_lock(&pmc_reserve_mutex);
 264                 if (atomic_read(&active_counters) == 0 && !reserve_pmc_hardware())
 265                         err = -EBUSY;
 266                 else
 267                         atomic_inc(&active_counters);
 268                 mutex_unlock(&pmc_reserve_mutex);
 269         }
 270         if (err)
 271                 return err;
 272
 273         /*
 274          * Generate PMC IRQs:
 275          * (keep 'enabled' bit clear for now)
 276          */
 277         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 278
 279         /*
 280          * Count user and OS events unless requested not to.
 281          */
 282         if (!hw_event->exclude_user)
 283                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 284         if (!hw_event->exclude_kernel)
 285                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 286
 287         /*
 288          * If privileged enough, allow NMI events:
 289          */
 290         hwc->nmi = 0;
 291         if (hw_event->nmi) {
 292                 if (sysctl_perf_counter_priv && !capable(CAP_SYS_ADMIN))
 293                         return -EACCES;
 294                 hwc->nmi = 1;
 295         }
 296         perf_counters_lapic_init(hwc->nmi);
 297
 298         if (!hwc->irq_period)
 299                 hwc->irq_period = x86_pmu.max_period;
 300
 301         atomic64_set(&hwc->period_left,
 302                         min(x86_pmu.max_period, hwc->irq_period));
 303
 304         /*
 305          * Raw event type provide the config in the event structure
 306          */
 307         if (perf_event_raw(hw_event)) {
 308                 hwc->config |= x86_pmu.raw_event(perf_event_config(hw_event));
 309         } else {
 310                 if (perf_event_id(hw_event) >= x86_pmu.max_events)
 311                         return -EINVAL;
 312                 /*
 313                  * The generic map:
 314                  */
 315                 hwc->config |= x86_pmu.event_map(perf_event_id(hw_event));
 316         }
 317
 318         counter->destroy = hw_perf_counter_destroy;
 319
 320         return 0;
 321 }
 322
 323 static void intel_pmu_disable_all(void)
 324 {
 325         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 326 }
 327
 328 static void amd_pmu_disable_all(void)
 329 {
 330         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 331         int idx;
 332
 333         if (!cpuc->enabled)
 334                 return;
 335
 336         cpuc->enabled = 0;
 337         /*
 338          * ensure we write the disable before we start disabling the
 339          * counters proper, so that amd_pmu_enable_counter() does the
 340          * right thing.
 341          */
 342         barrier();
 343
 344         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 345                 u64 val;
 346
 347                 if (!test_bit(idx, cpuc->active_mask))
 348                         continue;
 349                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 350                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
 351                         continue;
 352                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 353                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 354         }
 355 }
 356
 357 void hw_perf_disable(void)
 358 {
 359         if (!x86_pmu_initialized())
 360                 return;
 361         return x86_pmu.disable_all();
 362 }
 363
 364 static void intel_pmu_enable_all(void)
 365 {
 366         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
 367 }
 368
 369 static void amd_pmu_enable_all(void)
 370 {
 371         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 372         int idx;
 373
 374         if (cpuc->enabled)
 375                 return;
 376
 377         cpuc->enabled = 1;
 378         barrier();
 379
 380         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 381                 u64 val;
 382
 383                 if (!test_bit(idx, cpuc->active_mask))
 384                         continue;
 385                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 386                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE)
 387                         continue;
 388                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 389                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 390         }
 391 }
 392
 393 void hw_perf_enable(void)
 394 {
 395         if (!x86_pmu_initialized())
 396                 return;
 397         x86_pmu.enable_all();
 398 }
 399
 400 static inline u64 intel_pmu_get_status(void)
 401 {
 402         u64 status;
 403
 404         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 405
 406         return status;
 407 }
 408
 409 static inline void intel_pmu_ack_status(u64 ack)
 410 {
 411         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 412 }
 413
 414 static inline void x86_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 415 {
 416         int err;
 417         err = checking_wrmsrl(hwc->config_base + idx,
 418                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 419 }
 420
 421 static inline void x86_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 422 {
 423         int err;
 424         err = checking_wrmsrl(hwc->config_base + idx,
 425                               hwc->config);
 426 }
 427
 428 static inline void
 429 intel_pmu_disable_fixed(struct hw_perf_counter *hwc, int __idx)
 430 {
 431         int idx = __idx - X86_PMC_IDX_FIXED;
 432         u64 ctrl_val, mask;
 433         int err;
 434
 435         mask = 0xfULL << (idx * 4);
 436
 437         rdmsrl(hwc->config_base, ctrl_val);
 438         ctrl_val &= ~mask;
 439         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 440 }
 441
 442 static inline void
 443 intel_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 444 {
 445         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 446                 intel_pmu_disable_fixed(hwc, idx);
 447                 return;
 448         }
 449
 450         x86_pmu_disable_counter(hwc, idx);
 451 }
 452
 453 static inline void
 454 amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
 455 {
 456         x86_pmu_disable_counter(hwc, idx);
 457 }
 458
 459 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 460
 461 /*
 462  * Set the next IRQ period, based on the hwc->period_left value.
 463  * To be called with the counter disabled in hw:
 464  */
 465 static void
 466 x86_perf_counter_set_period(struct perf_counter *counter,
 467                              struct hw_perf_counter *hwc, int idx)
 468 {
 469         s64 left = atomic64_read(&hwc->period_left);
 470         s64 period = min(x86_pmu.max_period, hwc->irq_period);
 471         int err;
 472
 473         /*
 474          * If we are way outside a reasoable range then just skip forward:
 475          */
 476         if (unlikely(left <= -period)) {
 477                 left = period;
 478                 atomic64_set(&hwc->period_left, left);
 479         }
 480
 481         if (unlikely(left <= 0)) {
 482                 left += period;
 483                 atomic64_set(&hwc->period_left, left);
 484         }
 485         /*
 486          * Quirk: certain CPUs dont like it if just 1 event is left:
 487          */
 488         if (unlikely(left < 2))
 489                 left = 2;
 490
 491         per_cpu(prev_left[idx], smp_processor_id()) = left;
 492
 493         /*
 494          * The hw counter starts counting from this counter offset,
 495          * mark it to be able to extra future deltas:
 496          */
 497         atomic64_set(&hwc->prev_count, (u64)-left);
 498
 499         err = checking_wrmsrl(hwc->counter_base + idx,
 500                              (u64)(-left) & x86_pmu.counter_mask);
 501 }
 502
 503 static inline void
 504 intel_pmu_enable_fixed(struct hw_perf_counter *hwc, int __idx)
 505 {
 506         int idx = __idx - X86_PMC_IDX_FIXED;
 507         u64 ctrl_val, bits, mask;
 508         int err;
 509
 510         /*
 511          * Enable IRQ generation (0x8),
 512          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 513          * if requested:
 514          */
 515         bits = 0x8ULL;
 516         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 517                 bits |= 0x2;
 518         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 519                 bits |= 0x1;
 520         bits <<= (idx * 4);
 521         mask = 0xfULL << (idx * 4);
 522
 523         rdmsrl(hwc->config_base, ctrl_val);
 524         ctrl_val &= ~mask;
 525         ctrl_val |= bits;
 526         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 527 }
 528
 529 static void intel_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 530 {
 531         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
 532                 intel_pmu_enable_fixed(hwc, idx);
 533                 return;
 534         }
 535
 536         x86_pmu_enable_counter(hwc, idx);
 537 }
 538
 539 static void amd_pmu_enable_counter(struct hw_perf_counter *hwc, int idx)
 540 {
 541         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 542
 543         if (cpuc->enabled)
 544                 x86_pmu_enable_counter(hwc, idx);
 545         else
 546                 x86_pmu_disable_counter(hwc, idx);
 547 }
 548
 549 static int
 550 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 551 {
 552         unsigned int event;
 553
 554         if (!x86_pmu.num_counters_fixed)
 555                 return -1;
 556
 557         if (unlikely(hwc->nmi))
 558                 return -1;
 559
 560         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 561
 562         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_INSTRUCTIONS)))
 563                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 564         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_CPU_CYCLES)))
 565                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 566         if (unlikely(event == x86_pmu.event_map(PERF_COUNT_BUS_CYCLES)))
 567                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 568
 569         return -1;
 570 }
 571
 572 /*
 573  * Find a PMC slot for the freshly enabled / scheduled in counter:
 574  */
 575 static int x86_pmu_enable(struct perf_counter *counter)
 576 {
 577         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 578         struct hw_perf_counter *hwc = &counter->hw;
 579         int idx;
 580
 581         idx = fixed_mode_idx(counter, hwc);
 582         if (idx >= 0) {
 583                 /*
 584                  * Try to get the fixed counter, if that is already taken
 585                  * then try to get a generic counter:
 586                  */
 587                 if (test_and_set_bit(idx, cpuc->used_mask))
 588                         goto try_generic;
 589
 590                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 591                 /*
 592                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
 593                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
 594                  */
 595                 hwc->counter_base =
 596                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
 597                 hwc->idx = idx;
 598         } else {
 599                 idx = hwc->idx;
 600                 /* Try to get the previous generic counter again */
 601                 if (test_and_set_bit(idx, cpuc->used_mask)) {
 602 try_generic:
 603                         idx = find_first_zero_bit(cpuc->used_mask,
 604                                                   x86_pmu.num_counters);
 605                         if (idx == x86_pmu.num_counters)
 606                                 return -EAGAIN;
 607
 608                         set_bit(idx, cpuc->used_mask);
 609                         hwc->idx = idx;
 610                 }
 611                 hwc->config_base  = x86_pmu.eventsel;
 612                 hwc->counter_base = x86_pmu.perfctr;
 613         }
 614
 615         x86_pmu.disable(hwc, idx);
 616
 617         cpuc->counters[idx] = counter;
 618         set_bit(idx, cpuc->active_mask);
 619
 620         x86_perf_counter_set_period(counter, hwc, idx);
 621         x86_pmu.enable(hwc, idx);
 622
 623         return 0;
 624 }
 625
 626 void perf_counter_print_debug(void)
 627 {
 628         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
 629         struct cpu_hw_counters *cpuc;
 630         unsigned long flags;
 631         int cpu, idx;
 632
 633         if (!x86_pmu.num_counters)
 634                 return;
 635
 636         local_irq_save(flags);
 637
 638         cpu = smp_processor_id();
 639         cpuc = &per_cpu(cpu_hw_counters, cpu);
 640
 641         if (x86_pmu.version >= 2) {
 642                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 643                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 644                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
 645                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
 646
 647                 pr_info("\n");
 648                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
 649                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
 650                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
 651                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
 652         }
 653         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
 654
 655         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 656                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
 657                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
 658
 659                 prev_left = per_cpu(prev_left[idx], cpu);
 660
 661                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
 662                         cpu, idx, pmc_ctrl);
 663                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
 664                         cpu, idx, pmc_count);
 665                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
 666                         cpu, idx, prev_left);
 667         }
 668         for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
 669                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
 670
 671                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
 672                         cpu, idx, pmc_count);
 673         }
 674         local_irq_restore(flags);
 675 }
 676
 677 static void x86_pmu_disable(struct perf_counter *counter)
 678 {
 679         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 680         struct hw_perf_counter *hwc = &counter->hw;
 681         int idx = hwc->idx;
 682
 683         /*
 684          * Must be done before we disable, otherwise the nmi handler
 685          * could reenable again:
 686          */
 687         clear_bit(idx, cpuc->active_mask);
 688         x86_pmu.disable(hwc, idx);
 689
 690         /*
 691          * Make sure the cleared pointer becomes visible before we
 692          * (potentially) free the counter:
 693          */
 694         barrier();
 695
 696         /*
 697          * Drain the remaining delta count out of a counter
 698          * that we are disabling:
 699          */
 700         x86_perf_counter_update(counter, hwc, idx);
 701         cpuc->counters[idx] = NULL;
 702         clear_bit(idx, cpuc->used_mask);
 703 }
 704
 705 /*
 706  * Save and restart an expired counter. Called by NMI contexts,
 707  * so it has to be careful about preempting normal counter ops:
 708  */
 709 static void intel_pmu_save_and_restart(struct perf_counter *counter)
 710 {
 711         struct hw_perf_counter *hwc = &counter->hw;
 712         int idx = hwc->idx;
 713
 714         x86_perf_counter_update(counter, hwc, idx);
 715         x86_perf_counter_set_period(counter, hwc, idx);
 716
 717         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
 718                 intel_pmu_enable_counter(hwc, idx);
 719 }
 720
 721 /*
 722  * Maximum interrupt frequency of 100KHz per CPU
 723  */
 724 #define PERFMON_MAX_INTERRUPTS (100000/HZ)
 725
 726 /*
 727  * This handler is triggered by the local APIC, so the APIC IRQ handling
 728  * rules apply:
 729  */
 730 static int intel_pmu_handle_irq(struct pt_regs *regs, int nmi)
 731 {
 732         struct cpu_hw_counters *cpuc;
 733         struct cpu_hw_counters;
 734         int bit, cpu, loops;
 735         u64 ack, status;
 736
 737         cpu = smp_processor_id();
 738         cpuc = &per_cpu(cpu_hw_counters, cpu);
 739
 740         perf_disable();
 741         status = intel_pmu_get_status();
 742         if (!status) {
 743                 perf_enable();
 744                 return 0;
 745         }
 746
 747         loops = 0;
 748 again:
 749         if (++loops > 100) {
 750                 WARN_ONCE(1, "perfcounters: irq loop stuck!\n");
 751                 perf_counter_print_debug();
 752                 return 1;
 753         }
 754
 755         inc_irq_stat(apic_perf_irqs);
 756         ack = status;
 757         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 758                 struct perf_counter *counter = cpuc->counters[bit];
 759
 760                 clear_bit(bit, (unsigned long *) &status);
 761                 if (!test_bit(bit, cpuc->active_mask))
 762                         continue;
 763
 764                 intel_pmu_save_and_restart(counter);
 765                 if (perf_counter_overflow(counter, nmi, regs, 0))
 766                         intel_pmu_disable_counter(&counter->hw, bit);
 767         }
 768
 769         intel_pmu_ack_status(ack);
 770
 771         /*
 772          * Repeat if there is more work to be done:
 773          */
 774         status = intel_pmu_get_status();
 775         if (status)
 776                 goto again;
 777
 778         if (++cpuc->interrupts != PERFMON_MAX_INTERRUPTS)
 779                 perf_enable();
 780
 781         return 1;
 782 }
 783
 784 static int amd_pmu_handle_irq(struct pt_regs *regs, int nmi)
 785 {
 786         int cpu, idx, throttle = 0, handled = 0;
 787         struct cpu_hw_counters *cpuc;
 788         struct perf_counter *counter;
 789         struct hw_perf_counter *hwc;
 790         u64 val;
 791
 792         cpu = smp_processor_id();
 793         cpuc = &per_cpu(cpu_hw_counters, cpu);
 794
 795         if (++cpuc->interrupts == PERFMON_MAX_INTERRUPTS) {
 796                 throttle = 1;
 797                 __perf_disable();
 798                 cpuc->enabled = 0;
 799                 barrier();
 800         }
 801
 802         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 803                 int disable = 0;
 804
 805                 if (!test_bit(idx, cpuc->active_mask))
 806                         continue;
 807
 808                 counter = cpuc->counters[idx];
 809                 hwc = &counter->hw;
 810
 811                 if (counter->hw_event.nmi != nmi)
 812                         goto next;
 813
 814                 val = x86_perf_counter_update(counter, hwc, idx);
 815                 if (val & (1ULL << (x86_pmu.counter_bits - 1)))
 816                         goto next;
 817
 818                 /* counter overflow */
 819                 x86_perf_counter_set_period(counter, hwc, idx);
 820                 handled = 1;
 821                 inc_irq_stat(apic_perf_irqs);
 822                 disable = perf_counter_overflow(counter, nmi, regs, 0);
 823
 824 next:
 825                 if (disable || throttle)
 826                         amd_pmu_disable_counter(hwc, idx);
 827         }
 828
 829         return handled;
 830 }
 831
 832 void perf_counter_unthrottle(void)
 833 {
 834         struct cpu_hw_counters *cpuc;
 835
 836         if (!x86_pmu_initialized())
 837                 return;
 838
 839         cpuc = &__get_cpu_var(cpu_hw_counters);
 840         if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
 841                 /*
 842                  * Clear them before re-enabling irqs/NMIs again:
 843                  */
 844                 cpuc->interrupts = 0;
 845                 perf_enable();
 846         } else {
 847                 cpuc->interrupts = 0;
 848         }
 849 }
 850
 851 void smp_perf_counter_interrupt(struct pt_regs *regs)
 852 {
 853         irq_enter();
 854         apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 855         ack_APIC_irq();
 856         x86_pmu.handle_irq(regs, 0);
 857         irq_exit();
 858 }
 859
 860 void smp_perf_pending_interrupt(struct pt_regs *regs)
 861 {
 862         irq_enter();
 863         ack_APIC_irq();
 864         inc_irq_stat(apic_pending_irqs);
 865         perf_counter_do_pending();
 866         irq_exit();
 867 }
 868
 869 void set_perf_counter_pending(void)
 870 {
 871         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
 872 }
 873
 874 void perf_counters_lapic_init(int nmi)
 875 {
 876         u32 apic_val;
 877
 878         if (!x86_pmu_initialized())
 879                 return;
 880
 881         /*
 882          * Enable the performance counter vector in the APIC LVT:
 883          */
 884         apic_val = apic_read(APIC_LVTERR);
 885
 886         apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
 887         if (nmi)
 888                 apic_write(APIC_LVTPC, APIC_DM_NMI);
 889         else
 890                 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 891         apic_write(APIC_LVTERR, apic_val);
 892 }
 893
 894 static int __kprobes
 895 perf_counter_nmi_handler(struct notifier_block *self,
 896                          unsigned long cmd, void *__args)
 897 {
 898         struct die_args *args = __args;
 899         struct pt_regs *regs;
 900
 901         if (!atomic_read(&active_counters))
 902                 return NOTIFY_DONE;
 903
 904         switch (cmd) {
 905         case DIE_NMI:
 906         case DIE_NMI_IPI:
 907                 break;
 908
 909         default:
 910                 return NOTIFY_DONE;
 911         }
 912
 913         regs = args->regs;
 914
 915         apic_write(APIC_LVTPC, APIC_DM_NMI);
 916         /*
 917          * Can't rely on the handled return value to say it was our NMI, two
 918          * counters could trigger 'simultaneously' raising two back-to-back NMIs.
 919          *
 920          * If the first NMI handles both, the latter will be empty and daze
 921          * the CPU.
 922          */
 923         x86_pmu.handle_irq(regs, 1);
 924
 925         return NOTIFY_STOP;
 926 }
 927
 928 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
 929         .notifier_call          = perf_counter_nmi_handler,
 930         .next                   = NULL,
 931         .priority               = 1
 932 };
 933
 934 static struct x86_pmu intel_pmu = {
 935         .name                   = "Intel",
 936         .handle_irq             = intel_pmu_handle_irq,
 937         .disable_all            = intel_pmu_disable_all,
 938         .enable_all             = intel_pmu_enable_all,
 939         .enable                 = intel_pmu_enable_counter,
 940         .disable                = intel_pmu_disable_counter,
 941         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
 942         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
 943         .event_map              = intel_pmu_event_map,
 944         .raw_event              = intel_pmu_raw_event,
 945         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
 946         /*
 947          * Intel PMCs cannot be accessed sanely above 32 bit width,
 948          * so we install an artificial 1<<31 period regardless of
 949          * the generic counter period:
 950          */
 951         .max_period             = (1ULL << 31) - 1,
 952 };
 953
 954 static struct x86_pmu amd_pmu = {
 955         .name                   = "AMD",
 956         .handle_irq             = amd_pmu_handle_irq,
 957         .disable_all            = amd_pmu_disable_all,
 958         .enable_all             = amd_pmu_enable_all,
 959         .enable                 = amd_pmu_enable_counter,
 960         .disable                = amd_pmu_disable_counter,
 961         .eventsel               = MSR_K7_EVNTSEL0,
 962         .perfctr                = MSR_K7_PERFCTR0,
 963         .event_map              = amd_pmu_event_map,
 964         .raw_event              = amd_pmu_raw_event,
 965         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
 966         .num_counters           = 4,
 967         .counter_bits           = 48,
 968         .counter_mask           = (1ULL << 48) - 1,
 969         /* use highest bit to detect overflow */
 970         .max_period             = (1ULL << 47) - 1,
 971 };
 972
 973 static int intel_pmu_init(void)
 974 {
 975         union cpuid10_edx edx;
 976         union cpuid10_eax eax;
 977         unsigned int unused;
 978         unsigned int ebx;
 979         int version;
 980
 981         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 982                 return -ENODEV;
 983
 984         /*
 985          * Check whether the Architectural PerfMon supports
 986          * Branch Misses Retired Event or not.
 987          */
 988         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
 989         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
 990                 return -ENODEV;
 991
 992         version = eax.split.version_id;
 993         if (version < 2)
 994                 return -ENODEV;
 995
 996         x86_pmu = intel_pmu;
 997         x86_pmu.version = version;
 998         x86_pmu.num_counters = eax.split.num_counters;
 999
1000         /*
1001          * Quirk: v2 perfmon does not report fixed-purpose counters, so
1002          * assume at least 3 counters:
1003          */
1004         x86_pmu.num_counters_fixed = max((int)edx.split.num_counters_fixed, 3);
1005
1006         x86_pmu.counter_bits = eax.split.bit_width;
1007         x86_pmu.counter_mask = (1ULL << eax.split.bit_width) - 1;
1008
1009         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1010
1011         return 0;
1012 }
1013
1014 static int amd_pmu_init(void)
1015 {
1016         x86_pmu = amd_pmu;
1017         return 0;
1018 }
1019
1020 void __init init_hw_perf_counters(void)
1021 {
1022         int err;
1023
1024         switch (boot_cpu_data.x86_vendor) {
1025         case X86_VENDOR_INTEL:
1026                 err = intel_pmu_init();
1027                 break;
1028         case X86_VENDOR_AMD:
1029                 err = amd_pmu_init();
1030                 break;
1031         default:
1032                 return;
1033         }
1034         if (err != 0)
1035                 return;
1036
1037         pr_info("%s Performance Monitoring support detected.\n", x86_pmu.name);
1038         pr_info("... version:         %d\n", x86_pmu.version);
1039         pr_info("... bit width:       %d\n", x86_pmu.counter_bits);
1040
1041         pr_info("... num counters:    %d\n", x86_pmu.num_counters);
1042         if (x86_pmu.num_counters > X86_PMC_MAX_GENERIC) {
1043                 x86_pmu.num_counters = X86_PMC_MAX_GENERIC;
1044                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1045                      x86_pmu.num_counters, X86_PMC_MAX_GENERIC);
1046         }
1047         perf_counter_mask = (1 << x86_pmu.num_counters) - 1;
1048         perf_max_counters = x86_pmu.num_counters;
1049
1050         pr_info("... value mask:      %016Lx\n", x86_pmu.counter_mask);
1051         pr_info("... max period:      %016Lx\n", x86_pmu.max_period);
1052
1053         if (x86_pmu.num_counters_fixed > X86_PMC_MAX_FIXED) {
1054                 x86_pmu.num_counters_fixed = X86_PMC_MAX_FIXED;
1055                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1056                      x86_pmu.num_counters_fixed, X86_PMC_MAX_FIXED);
1057         }
1058         pr_info("... fixed counters:  %d\n", x86_pmu.num_counters_fixed);
1059
1060         perf_counter_mask |=
1061                 ((1LL << x86_pmu.num_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1062
1063         pr_info("... counter mask:    %016Lx\n", perf_counter_mask);
1064
1065         perf_counters_lapic_init(1);
1066         register_die_notifier(&perf_counter_nmi_notifier);
1067 }
1068
1069 static inline void x86_pmu_read(struct perf_counter *counter)
1070 {
1071         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1072 }
1073
1074 static const struct pmu pmu = {
1075         .enable         = x86_pmu_enable,
1076         .disable        = x86_pmu_disable,
1077         .read           = x86_pmu_read,
1078 };
1079
1080 const struct pmu *hw_perf_counter_init(struct perf_counter *counter)
1081 {
1082         int err;
1083
1084         err = __hw_perf_counter_init(counter);
1085         if (err)
1086                 return ERR_PTR(err);
1087
1088         return &pmu;
1089 }
1090
1091 /*
1092  * callchain support
1093  */
1094
1095 static inline
1096 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1097 {
1098         if (entry->nr < MAX_STACK_DEPTH)
1099                 entry->ip[entry->nr++] = ip;
1100 }
1101
1102 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1103 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1104
1105
1106 static void
1107 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1108 {
1109         /* Ignore warnings */
1110 }
1111
1112 static void backtrace_warning(void *data, char *msg)
1113 {
1114         /* Ignore warnings */
1115 }
1116
1117 static int backtrace_stack(void *data, char *name)
1118 {
1119         /* Don't bother with IRQ stacks for now */
1120         return -1;
1121 }
1122
1123 static void backtrace_address(void *data, unsigned long addr, int reliable)
1124 {
1125         struct perf_callchain_entry *entry = data;
1126
1127         if (reliable)
1128                 callchain_store(entry, addr);
1129 }
1130
1131 static const struct stacktrace_ops backtrace_ops = {
1132         .warning                = backtrace_warning,
1133         .warning_symbol         = backtrace_warning_symbol,
1134         .stack                  = backtrace_stack,
1135         .address                = backtrace_address,
1136 };
1137
1138 static void
1139 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1140 {
1141         unsigned long bp;
1142         char *stack;
1143         int nr = entry->nr;
1144
1145         callchain_store(entry, instruction_pointer(regs));
1146
1147         stack = ((char *)regs + sizeof(struct pt_regs));
1148 #ifdef CONFIG_FRAME_POINTER
1149         bp = frame_pointer(regs);
1150 #else
1151         bp = 0;
1152 #endif
1153
1154         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1155
1156         entry->kernel = entry->nr - nr;
1157 }
1158
1159
1160 struct stack_frame {
1161         const void __user       *next_fp;
1162         unsigned long           return_address;
1163 };
1164
1165 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1166 {
1167         int ret;
1168
1169         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1170                 return 0;
1171
1172         ret = 1;
1173         pagefault_disable();
1174         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1175                 ret = 0;
1176         pagefault_enable();
1177
1178         return ret;
1179 }
1180
1181 static void
1182 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1183 {
1184         struct stack_frame frame;
1185         const void __user *fp;
1186         int nr = entry->nr;
1187
1188         regs = (struct pt_regs *)current->thread.sp0 - 1;
1189         fp   = (void __user *)regs->bp;
1190
1191         callchain_store(entry, regs->ip);
1192
1193         while (entry->nr < MAX_STACK_DEPTH) {
1194                 frame.next_fp        = NULL;
1195                 frame.return_address = 0;
1196
1197                 if (!copy_stack_frame(fp, &frame))
1198                         break;
1199
1200                 if ((unsigned long)fp < user_stack_pointer(regs))
1201                         break;
1202
1203                 callchain_store(entry, frame.return_address);
1204                 fp = frame.next_fp;
1205         }
1206
1207         entry->user = entry->nr - nr;
1208 }
1209
1210 static void
1211 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1212 {
1213         int is_user;
1214
1215         if (!regs)
1216                 return;
1217
1218         is_user = user_mode(regs);
1219
1220         if (!current || current->pid == 0)
1221                 return;
1222
1223         if (is_user && current->state != TASK_RUNNING)
1224                 return;
1225
1226         if (!is_user)
1227                 perf_callchain_kernel(regs, entry);
1228
1229         if (current->mm)
1230                 perf_callchain_user(regs, entry);
1231 }
1232
1233 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1234 {
1235         struct perf_callchain_entry *entry;
1236
1237         if (in_nmi())
1238                 entry = &__get_cpu_var(nmi_entry);
1239         else
1240                 entry = &__get_cpu_var(irq_entry);
1241
1242         entry->nr = 0;
1243         entry->hv = 0;
1244         entry->kernel = 0;
1245         entry->user = 0;
1246
1247         perf_do_callchain(regs, entry);
1248
1249         return entry;
1250 }