arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
   6  *  Copyright(C) 2009 Jaswinder Singh Rajput
   7  *
   8  *  For licencing details see kernel-base/COPYING
   9  */
  10
  11 #include <linux/perf_counter.h>
  12 #include <linux/capability.h>
  13 #include <linux/notifier.h>
  14 #include <linux/hardirq.h>
  15 #include <linux/kprobes.h>
  16 #include <linux/module.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/sched.h>
  19 #include <linux/uaccess.h>
  20
  21 #include <asm/apic.h>
  22 #include <asm/stacktrace.h>
  23 #include <asm/nmi.h>
  24
  25 static bool perf_counters_initialized __read_mostly;
  26
  27 /*
  28  * Number of (generic) HW counters:
  29  */
  30 static int nr_counters_generic __read_mostly;
  31 static u64 perf_counter_mask __read_mostly;
  32 static u64 counter_value_mask __read_mostly;
  33 static int counter_value_bits __read_mostly;
  34
  35 static int nr_counters_fixed __read_mostly;
  36
  37 struct cpu_hw_counters {
  38         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  39         unsigned long           used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  40         unsigned long           interrupts;
  41         u64                     throttle_ctrl;
  42         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  43         int                     enabled;
  44 };
  45
  46 /*
  47  * struct pmc_x86_ops - performance counter x86 ops
  48  */
  49 struct pmc_x86_ops {
  50         u64             (*save_disable_all)(void);
  51         void            (*restore_all)(u64);
  52         u64             (*get_status)(u64);
  53         void            (*ack_status)(u64);
  54         void            (*enable)(int, u64);
  55         void            (*disable)(int, u64);
  56         unsigned        eventsel;
  57         unsigned        perfctr;
  58         u64             (*event_map)(int);
  59         u64             (*raw_event)(u64);
  60         int             max_events;
  61 };
  62
  63 static struct pmc_x86_ops *pmc_ops __read_mostly;
  64
  65 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  66         .enabled = 1,
  67 };
  68
  69 static __read_mostly int intel_perfmon_version;
  70
  71 /*
  72  * Intel PerfMon v3. Used on Core2 and later.
  73  */
  74 static const u64 intel_perfmon_event_map[] =
  75 {
  76   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  77   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  78   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  79   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  80   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  81   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  82   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  83 };
  84
  85 static u64 pmc_intel_event_map(int event)
  86 {
  87         return intel_perfmon_event_map[event];
  88 }
  89
  90 static u64 pmc_intel_raw_event(u64 event)
  91 {
  92 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
  93 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
  94 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
  95
  96 #define CORE_EVNTSEL_MASK               \
  97         (CORE_EVNTSEL_EVENT_MASK |      \
  98          CORE_EVNTSEL_UNIT_MASK  |      \
  99          CORE_EVNTSEL_COUNTER_MASK)
 100
 101         return event & CORE_EVNTSEL_MASK;
 102 }
 103
 104 /*
 105  * AMD Performance Monitor K7 and later.
 106  */
 107 static const u64 amd_perfmon_event_map[] =
 108 {
 109   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
 110   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
 111   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
 112   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
 113   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
 114   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
 115 };
 116
 117 static u64 pmc_amd_event_map(int event)
 118 {
 119         return amd_perfmon_event_map[event];
 120 }
 121
 122 static u64 pmc_amd_raw_event(u64 event)
 123 {
 124 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 125 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 126 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 127
 128 #define K7_EVNTSEL_MASK                 \
 129         (K7_EVNTSEL_EVENT_MASK |        \
 130          K7_EVNTSEL_UNIT_MASK  |        \
 131          K7_EVNTSEL_COUNTER_MASK)
 132
 133         return event & K7_EVNTSEL_MASK;
 134 }
 135
 136 /*
 137  * Propagate counter elapsed time into the generic counter.
 138  * Can only be executed on the CPU where the counter is active.
 139  * Returns the delta events processed.
 140  */
 141 static void
 142 x86_perf_counter_update(struct perf_counter *counter,
 143                         struct hw_perf_counter *hwc, int idx)
 144 {
 145         u64 prev_raw_count, new_raw_count, delta;
 146
 147         /*
 148          * Careful: an NMI might modify the previous counter value.
 149          *
 150          * Our tactic to handle this is to first atomically read and
 151          * exchange a new raw count - then add that new-prev delta
 152          * count to the generic counter atomically:
 153          */
 154 again:
 155         prev_raw_count = atomic64_read(&hwc->prev_count);
 156         rdmsrl(hwc->counter_base + idx, new_raw_count);
 157
 158         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 159                                         new_raw_count) != prev_raw_count)
 160                 goto again;
 161
 162         /*
 163          * Now we have the new raw value and have updated the prev
 164          * timestamp already. We can now calculate the elapsed delta
 165          * (counter-)time and add that to the generic counter.
 166          *
 167          * Careful, not all hw sign-extends above the physical width
 168          * of the count, so we do that by clipping the delta to 32 bits:
 169          */
 170         delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
 171
 172         atomic64_add(delta, &counter->count);
 173         atomic64_sub(delta, &hwc->period_left);
 174 }
 175
 176 static atomic_t num_counters;
 177 static DEFINE_MUTEX(pmc_reserve_mutex);
 178
 179 static bool reserve_pmc_hardware(void)
 180 {
 181         int i;
 182
 183         if (nmi_watchdog == NMI_LOCAL_APIC)
 184                 disable_lapic_nmi_watchdog();
 185
 186         for (i = 0; i < nr_counters_generic; i++) {
 187                 if (!reserve_perfctr_nmi(pmc_ops->perfctr + i))
 188                         goto perfctr_fail;
 189         }
 190
 191         for (i = 0; i < nr_counters_generic; i++) {
 192                 if (!reserve_evntsel_nmi(pmc_ops->eventsel + i))
 193                         goto eventsel_fail;
 194         }
 195
 196         return true;
 197
 198 eventsel_fail:
 199         for (i--; i >= 0; i--)
 200                 release_evntsel_nmi(pmc_ops->eventsel + i);
 201
 202         i = nr_counters_generic;
 203
 204 perfctr_fail:
 205         for (i--; i >= 0; i--)
 206                 release_perfctr_nmi(pmc_ops->perfctr + i);
 207
 208         if (nmi_watchdog == NMI_LOCAL_APIC)
 209                 enable_lapic_nmi_watchdog();
 210
 211         return false;
 212 }
 213
 214 static void release_pmc_hardware(void)
 215 {
 216         int i;
 217
 218         for (i = 0; i < nr_counters_generic; i++) {
 219                 release_perfctr_nmi(pmc_ops->perfctr + i);
 220                 release_evntsel_nmi(pmc_ops->eventsel + i);
 221         }
 222
 223         if (nmi_watchdog == NMI_LOCAL_APIC)
 224                 enable_lapic_nmi_watchdog();
 225 }
 226
 227 static void hw_perf_counter_destroy(struct perf_counter *counter)
 228 {
 229         if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
 230                 release_pmc_hardware();
 231                 mutex_unlock(&pmc_reserve_mutex);
 232         }
 233 }
 234
 235 /*
 236  * Setup the hardware configuration for a given hw_event_type
 237  */
 238 static int __hw_perf_counter_init(struct perf_counter *counter)
 239 {
 240         struct perf_counter_hw_event *hw_event = &counter->hw_event;
 241         struct hw_perf_counter *hwc = &counter->hw;
 242         int err;
 243
 244         if (unlikely(!perf_counters_initialized))
 245                 return -EINVAL;
 246
 247         err = 0;
 248         if (atomic_inc_not_zero(&num_counters)) {
 249                 mutex_lock(&pmc_reserve_mutex);
 250                 if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
 251                         err = -EBUSY;
 252                 else
 253                         atomic_inc(&num_counters);
 254                 mutex_unlock(&pmc_reserve_mutex);
 255         }
 256         if (err)
 257                 return err;
 258
 259         /*
 260          * Generate PMC IRQs:
 261          * (keep 'enabled' bit clear for now)
 262          */
 263         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 264
 265         /*
 266          * Count user and OS events unless requested not to.
 267          */
 268         if (!hw_event->exclude_user)
 269                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 270         if (!hw_event->exclude_kernel)
 271                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 272
 273         /*
 274          * If privileged enough, allow NMI events:
 275          */
 276         hwc->nmi = 0;
 277         if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
 278                 hwc->nmi = 1;
 279
 280         hwc->irq_period         = hw_event->irq_period;
 281         /*
 282          * Intel PMCs cannot be accessed sanely above 32 bit width,
 283          * so we install an artificial 1<<31 period regardless of
 284          * the generic counter period:
 285          */
 286         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 287                 if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
 288                         hwc->irq_period = 0x7FFFFFFF;
 289
 290         atomic64_set(&hwc->period_left, hwc->irq_period);
 291
 292         /*
 293          * Raw event type provide the config in the event structure
 294          */
 295         if (perf_event_raw(hw_event)) {
 296                 hwc->config |= pmc_ops->raw_event(perf_event_config(hw_event));
 297         } else {
 298                 if (perf_event_id(hw_event) >= pmc_ops->max_events)
 299                         return -EINVAL;
 300                 /*
 301                  * The generic map:
 302                  */
 303                 hwc->config |= pmc_ops->event_map(perf_event_id(hw_event));
 304         }
 305
 306         counter->destroy = hw_perf_counter_destroy;
 307
 308         return 0;
 309 }
 310
 311 static u64 pmc_intel_save_disable_all(void)
 312 {
 313         u64 ctrl;
 314
 315         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 316         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 317
 318         return ctrl;
 319 }
 320
 321 static u64 pmc_amd_save_disable_all(void)
 322 {
 323         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 324         int enabled, idx;
 325
 326         enabled = cpuc->enabled;
 327         cpuc->enabled = 0;
 328         /*
 329          * ensure we write the disable before we start disabling the
 330          * counters proper, so that pcm_amd_enable() does the right thing.
 331          */
 332         barrier();
 333
 334         for (idx = 0; idx < nr_counters_generic; idx++) {
 335                 u64 val;
 336
 337                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 338                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE) {
 339                         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 340                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 341                 }
 342         }
 343
 344         return enabled;
 345 }
 346
 347 u64 hw_perf_save_disable(void)
 348 {
 349         if (unlikely(!perf_counters_initialized))
 350                 return 0;
 351
 352         return pmc_ops->save_disable_all();
 353 }
 354 /*
 355  * Exported because of ACPI idle
 356  */
 357 EXPORT_SYMBOL_GPL(hw_perf_save_disable);
 358
 359 static void pmc_intel_restore_all(u64 ctrl)
 360 {
 361         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 362 }
 363
 364 static void pmc_amd_restore_all(u64 ctrl)
 365 {
 366         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 367         int idx;
 368
 369         cpuc->enabled = ctrl;
 370         barrier();
 371         if (!ctrl)
 372                 return;
 373
 374         for (idx = 0; idx < nr_counters_generic; idx++) {
 375                 if (test_bit(idx, cpuc->active_mask)) {
 376                         u64 val;
 377
 378                         rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 379                         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 380                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 381                 }
 382         }
 383 }
 384
 385 void hw_perf_restore(u64 ctrl)
 386 {
 387         if (unlikely(!perf_counters_initialized))
 388                 return;
 389
 390         pmc_ops->restore_all(ctrl);
 391 }
 392 /*
 393  * Exported because of ACPI idle
 394  */
 395 EXPORT_SYMBOL_GPL(hw_perf_restore);
 396
 397 static u64 pmc_intel_get_status(u64 mask)
 398 {
 399         u64 status;
 400
 401         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 402
 403         return status;
 404 }
 405
 406 static u64 pmc_amd_get_status(u64 mask)
 407 {
 408         u64 status = 0;
 409         int idx;
 410
 411         for (idx = 0; idx < nr_counters_generic; idx++) {
 412                 s64 val;
 413
 414                 if (!(mask & (1 << idx)))
 415                         continue;
 416
 417                 rdmsrl(MSR_K7_PERFCTR0 + idx, val);
 418                 val <<= (64 - counter_value_bits);
 419                 if (val >= 0)
 420                         status |= (1 << idx);
 421         }
 422
 423         return status;
 424 }
 425
 426 static u64 hw_perf_get_status(u64 mask)
 427 {
 428         if (unlikely(!perf_counters_initialized))
 429                 return 0;
 430
 431         return pmc_ops->get_status(mask);
 432 }
 433
 434 static void pmc_intel_ack_status(u64 ack)
 435 {
 436         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 437 }
 438
 439 static void pmc_amd_ack_status(u64 ack)
 440 {
 441 }
 442
 443 static void hw_perf_ack_status(u64 ack)
 444 {
 445         if (unlikely(!perf_counters_initialized))
 446                 return;
 447
 448         pmc_ops->ack_status(ack);
 449 }
 450
 451 static void pmc_intel_enable(int idx, u64 config)
 452 {
 453         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx,
 454                         config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 455 }
 456
 457 static void pmc_amd_enable(int idx, u64 config)
 458 {
 459         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 460
 461         set_bit(idx, cpuc->active_mask);
 462         if (cpuc->enabled)
 463                 config |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 464
 465         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 466 }
 467
 468 static void hw_perf_enable(int idx, u64 config)
 469 {
 470         if (unlikely(!perf_counters_initialized))
 471                 return;
 472
 473         pmc_ops->enable(idx, config);
 474 }
 475
 476 static void pmc_intel_disable(int idx, u64 config)
 477 {
 478         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, config);
 479 }
 480
 481 static void pmc_amd_disable(int idx, u64 config)
 482 {
 483         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 484
 485         clear_bit(idx, cpuc->active_mask);
 486         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 487
 488 }
 489
 490 static void hw_perf_disable(int idx, u64 config)
 491 {
 492         if (unlikely(!perf_counters_initialized))
 493                 return;
 494
 495         pmc_ops->disable(idx, config);
 496 }
 497
 498 static inline void
 499 __pmc_fixed_disable(struct perf_counter *counter,
 500                     struct hw_perf_counter *hwc, unsigned int __idx)
 501 {
 502         int idx = __idx - X86_PMC_IDX_FIXED;
 503         u64 ctrl_val, mask;
 504         int err;
 505
 506         mask = 0xfULL << (idx * 4);
 507
 508         rdmsrl(hwc->config_base, ctrl_val);
 509         ctrl_val &= ~mask;
 510         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 511 }
 512
 513 static inline void
 514 __pmc_generic_disable(struct perf_counter *counter,
 515                            struct hw_perf_counter *hwc, unsigned int idx)
 516 {
 517         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 518                 __pmc_fixed_disable(counter, hwc, idx);
 519         else
 520                 hw_perf_disable(idx, hwc->config);
 521 }
 522
 523 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 524
 525 /*
 526  * Set the next IRQ period, based on the hwc->period_left value.
 527  * To be called with the counter disabled in hw:
 528  */
 529 static void
 530 __hw_perf_counter_set_period(struct perf_counter *counter,
 531                              struct hw_perf_counter *hwc, int idx)
 532 {
 533         s64 left = atomic64_read(&hwc->period_left);
 534         s64 period = hwc->irq_period;
 535         int err;
 536
 537         /*
 538          * If we are way outside a reasoable range then just skip forward:
 539          */
 540         if (unlikely(left <= -period)) {
 541                 left = period;
 542                 atomic64_set(&hwc->period_left, left);
 543         }
 544
 545         if (unlikely(left <= 0)) {
 546                 left += period;
 547                 atomic64_set(&hwc->period_left, left);
 548         }
 549
 550         per_cpu(prev_left[idx], smp_processor_id()) = left;
 551
 552         /*
 553          * The hw counter starts counting from this counter offset,
 554          * mark it to be able to extra future deltas:
 555          */
 556         atomic64_set(&hwc->prev_count, (u64)-left);
 557
 558         err = checking_wrmsrl(hwc->counter_base + idx,
 559                              (u64)(-left) & counter_value_mask);
 560 }
 561
 562 static inline void
 563 __pmc_fixed_enable(struct perf_counter *counter,
 564                    struct hw_perf_counter *hwc, unsigned int __idx)
 565 {
 566         int idx = __idx - X86_PMC_IDX_FIXED;
 567         u64 ctrl_val, bits, mask;
 568         int err;
 569
 570         /*
 571          * Enable IRQ generation (0x8),
 572          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 573          * if requested:
 574          */
 575         bits = 0x8ULL;
 576         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 577                 bits |= 0x2;
 578         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 579                 bits |= 0x1;
 580         bits <<= (idx * 4);
 581         mask = 0xfULL << (idx * 4);
 582
 583         rdmsrl(hwc->config_base, ctrl_val);
 584         ctrl_val &= ~mask;
 585         ctrl_val |= bits;
 586         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 587 }
 588
 589 static void
 590 __pmc_generic_enable(struct perf_counter *counter,
 591                           struct hw_perf_counter *hwc, int idx)
 592 {
 593         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 594                 __pmc_fixed_enable(counter, hwc, idx);
 595         else
 596                 hw_perf_enable(idx, hwc->config);
 597 }
 598
 599 static int
 600 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 601 {
 602         unsigned int event;
 603
 604         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 605                 return -1;
 606
 607         if (unlikely(hwc->nmi))
 608                 return -1;
 609
 610         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 611
 612         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_INSTRUCTIONS)))
 613                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 614         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_CPU_CYCLES)))
 615                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 616         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_BUS_CYCLES)))
 617                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 618
 619         return -1;
 620 }
 621
 622 /*
 623  * Find a PMC slot for the freshly enabled / scheduled in counter:
 624  */
 625 static int pmc_generic_enable(struct perf_counter *counter)
 626 {
 627         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 628         struct hw_perf_counter *hwc = &counter->hw;
 629         int idx;
 630
 631         idx = fixed_mode_idx(counter, hwc);
 632         if (idx >= 0) {
 633                 /*
 634                  * Try to get the fixed counter, if that is already taken
 635                  * then try to get a generic counter:
 636                  */
 637                 if (test_and_set_bit(idx, cpuc->used))
 638                         goto try_generic;
 639
 640                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 641                 /*
 642                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
 643                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
 644                  */
 645                 hwc->counter_base =
 646                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
 647                 hwc->idx = idx;
 648         } else {
 649                 idx = hwc->idx;
 650                 /* Try to get the previous generic counter again */
 651                 if (test_and_set_bit(idx, cpuc->used)) {
 652 try_generic:
 653                         idx = find_first_zero_bit(cpuc->used, nr_counters_generic);
 654                         if (idx == nr_counters_generic)
 655                                 return -EAGAIN;
 656
 657                         set_bit(idx, cpuc->used);
 658                         hwc->idx = idx;
 659                 }
 660                 hwc->config_base  = pmc_ops->eventsel;
 661                 hwc->counter_base = pmc_ops->perfctr;
 662         }
 663
 664         perf_counters_lapic_init(hwc->nmi);
 665
 666         __pmc_generic_disable(counter, hwc, idx);
 667
 668         cpuc->counters[idx] = counter;
 669         /*
 670          * Make it visible before enabling the hw:
 671          */
 672         smp_wmb();
 673
 674         __hw_perf_counter_set_period(counter, hwc, idx);
 675         __pmc_generic_enable(counter, hwc, idx);
 676
 677         return 0;
 678 }
 679
 680 void perf_counter_print_debug(void)
 681 {
 682         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
 683         struct cpu_hw_counters *cpuc;
 684         int cpu, idx;
 685
 686         if (!nr_counters_generic)
 687                 return;
 688
 689         local_irq_disable();
 690
 691         cpu = smp_processor_id();
 692         cpuc = &per_cpu(cpu_hw_counters, cpu);
 693
 694         if (intel_perfmon_version >= 2) {
 695                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 696                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 697                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
 698                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
 699
 700                 pr_info("\n");
 701                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
 702                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
 703                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
 704                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
 705         }
 706         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
 707
 708         for (idx = 0; idx < nr_counters_generic; idx++) {
 709                 rdmsrl(pmc_ops->eventsel + idx, pmc_ctrl);
 710                 rdmsrl(pmc_ops->perfctr  + idx, pmc_count);
 711
 712                 prev_left = per_cpu(prev_left[idx], cpu);
 713
 714                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
 715                         cpu, idx, pmc_ctrl);
 716                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
 717                         cpu, idx, pmc_count);
 718                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
 719                         cpu, idx, prev_left);
 720         }
 721         for (idx = 0; idx < nr_counters_fixed; idx++) {
 722                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
 723
 724                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
 725                         cpu, idx, pmc_count);
 726         }
 727         local_irq_enable();
 728 }
 729
 730 static void pmc_generic_disable(struct perf_counter *counter)
 731 {
 732         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 733         struct hw_perf_counter *hwc = &counter->hw;
 734         unsigned int idx = hwc->idx;
 735
 736         __pmc_generic_disable(counter, hwc, idx);
 737
 738         clear_bit(idx, cpuc->used);
 739         cpuc->counters[idx] = NULL;
 740         /*
 741          * Make sure the cleared pointer becomes visible before we
 742          * (potentially) free the counter:
 743          */
 744         smp_wmb();
 745
 746         /*
 747          * Drain the remaining delta count out of a counter
 748          * that we are disabling:
 749          */
 750         x86_perf_counter_update(counter, hwc, idx);
 751 }
 752
 753 /*
 754  * Save and restart an expired counter. Called by NMI contexts,
 755  * so it has to be careful about preempting normal counter ops:
 756  */
 757 static void perf_save_and_restart(struct perf_counter *counter)
 758 {
 759         struct hw_perf_counter *hwc = &counter->hw;
 760         int idx = hwc->idx;
 761
 762         x86_perf_counter_update(counter, hwc, idx);
 763         __hw_perf_counter_set_period(counter, hwc, idx);
 764
 765         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
 766                 __pmc_generic_enable(counter, hwc, idx);
 767 }
 768
 769 /*
 770  * Maximum interrupt frequency of 100KHz per CPU
 771  */
 772 #define PERFMON_MAX_INTERRUPTS (100000/HZ)
 773
 774 /*
 775  * This handler is triggered by the local APIC, so the APIC IRQ handling
 776  * rules apply:
 777  */
 778 static int __smp_perf_counter_interrupt(struct pt_regs *regs, int nmi)
 779 {
 780         int bit, cpu = smp_processor_id();
 781         u64 ack, status;
 782         struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
 783         int ret = 0;
 784
 785         cpuc->throttle_ctrl = hw_perf_save_disable();
 786
 787         status = hw_perf_get_status(cpuc->throttle_ctrl);
 788         if (!status)
 789                 goto out;
 790
 791         ret = 1;
 792 again:
 793         inc_irq_stat(apic_perf_irqs);
 794         ack = status;
 795         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 796                 struct perf_counter *counter = cpuc->counters[bit];
 797
 798                 clear_bit(bit, (unsigned long *) &status);
 799                 if (!counter)
 800                         continue;
 801
 802                 perf_save_and_restart(counter);
 803                 if (perf_counter_overflow(counter, nmi, regs, 0))
 804                         __pmc_generic_disable(counter, &counter->hw, bit);
 805         }
 806
 807         hw_perf_ack_status(ack);
 808
 809         /*
 810          * Repeat if there is more work to be done:
 811          */
 812         status = hw_perf_get_status(cpuc->throttle_ctrl);
 813         if (status)
 814                 goto again;
 815 out:
 816         /*
 817          * Restore - do not reenable when global enable is off or throttled:
 818          */
 819         if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
 820                 hw_perf_restore(cpuc->throttle_ctrl);
 821
 822         return ret;
 823 }
 824
 825 void perf_counter_unthrottle(void)
 826 {
 827         struct cpu_hw_counters *cpuc;
 828
 829         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 830                 return;
 831
 832         if (unlikely(!perf_counters_initialized))
 833                 return;
 834
 835         cpuc = &__get_cpu_var(cpu_hw_counters);
 836         if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
 837                 if (printk_ratelimit())
 838                         printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
 839                 hw_perf_restore(cpuc->throttle_ctrl);
 840         }
 841         cpuc->interrupts = 0;
 842 }
 843
 844 void smp_perf_counter_interrupt(struct pt_regs *regs)
 845 {
 846         irq_enter();
 847         apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 848         ack_APIC_irq();
 849         __smp_perf_counter_interrupt(regs, 0);
 850         irq_exit();
 851 }
 852
 853 void smp_perf_pending_interrupt(struct pt_regs *regs)
 854 {
 855         irq_enter();
 856         ack_APIC_irq();
 857         inc_irq_stat(apic_pending_irqs);
 858         perf_counter_do_pending();
 859         irq_exit();
 860 }
 861
 862 void set_perf_counter_pending(void)
 863 {
 864         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
 865 }
 866
 867 void perf_counters_lapic_init(int nmi)
 868 {
 869         u32 apic_val;
 870
 871         if (!perf_counters_initialized)
 872                 return;
 873         /*
 874          * Enable the performance counter vector in the APIC LVT:
 875          */
 876         apic_val = apic_read(APIC_LVTERR);
 877
 878         apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
 879         if (nmi)
 880                 apic_write(APIC_LVTPC, APIC_DM_NMI);
 881         else
 882                 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 883         apic_write(APIC_LVTERR, apic_val);
 884 }
 885
 886 static int __kprobes
 887 perf_counter_nmi_handler(struct notifier_block *self,
 888                          unsigned long cmd, void *__args)
 889 {
 890         struct die_args *args = __args;
 891         struct pt_regs *regs;
 892         int ret;
 893
 894         switch (cmd) {
 895         case DIE_NMI:
 896         case DIE_NMI_IPI:
 897                 break;
 898
 899         default:
 900                 return NOTIFY_DONE;
 901         }
 902
 903         regs = args->regs;
 904
 905         apic_write(APIC_LVTPC, APIC_DM_NMI);
 906         ret = __smp_perf_counter_interrupt(regs, 1);
 907
 908         return ret ? NOTIFY_STOP : NOTIFY_OK;
 909 }
 910
 911 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
 912         .notifier_call          = perf_counter_nmi_handler,
 913         .next                   = NULL,
 914         .priority               = 1
 915 };
 916
 917 static struct pmc_x86_ops pmc_intel_ops = {
 918         .save_disable_all       = pmc_intel_save_disable_all,
 919         .restore_all            = pmc_intel_restore_all,
 920         .get_status             = pmc_intel_get_status,
 921         .ack_status             = pmc_intel_ack_status,
 922         .enable                 = pmc_intel_enable,
 923         .disable                = pmc_intel_disable,
 924         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
 925         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
 926         .event_map              = pmc_intel_event_map,
 927         .raw_event              = pmc_intel_raw_event,
 928         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
 929 };
 930
 931 static struct pmc_x86_ops pmc_amd_ops = {
 932         .save_disable_all       = pmc_amd_save_disable_all,
 933         .restore_all            = pmc_amd_restore_all,
 934         .get_status             = pmc_amd_get_status,
 935         .ack_status             = pmc_amd_ack_status,
 936         .enable                 = pmc_amd_enable,
 937         .disable                = pmc_amd_disable,
 938         .eventsel               = MSR_K7_EVNTSEL0,
 939         .perfctr                = MSR_K7_PERFCTR0,
 940         .event_map              = pmc_amd_event_map,
 941         .raw_event              = pmc_amd_raw_event,
 942         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
 943 };
 944
 945 static struct pmc_x86_ops *pmc_intel_init(void)
 946 {
 947         union cpuid10_edx edx;
 948         union cpuid10_eax eax;
 949         unsigned int unused;
 950         unsigned int ebx;
 951
 952         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 953                 return NULL;
 954
 955         /*
 956          * Check whether the Architectural PerfMon supports
 957          * Branch Misses Retired Event or not.
 958          */
 959         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
 960         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
 961                 return NULL;
 962
 963         intel_perfmon_version = eax.split.version_id;
 964         if (intel_perfmon_version < 2)
 965                 return NULL;
 966
 967         pr_info("Intel Performance Monitoring support detected.\n");
 968         pr_info("... version:         %d\n", intel_perfmon_version);
 969         pr_info("... bit width:       %d\n", eax.split.bit_width);
 970         pr_info("... mask length:     %d\n", eax.split.mask_length);
 971
 972         nr_counters_generic = eax.split.num_counters;
 973         nr_counters_fixed = edx.split.num_counters_fixed;
 974         counter_value_mask = (1ULL << eax.split.bit_width) - 1;
 975
 976         return &pmc_intel_ops;
 977 }
 978
 979 static struct pmc_x86_ops *pmc_amd_init(void)
 980 {
 981         nr_counters_generic = 4;
 982         nr_counters_fixed = 0;
 983         counter_value_mask = 0x0000FFFFFFFFFFFFULL;
 984         counter_value_bits = 48;
 985
 986         pr_info("AMD Performance Monitoring support detected.\n");
 987
 988         return &pmc_amd_ops;
 989 }
 990
 991 void __init init_hw_perf_counters(void)
 992 {
 993         switch (boot_cpu_data.x86_vendor) {
 994         case X86_VENDOR_INTEL:
 995                 pmc_ops = pmc_intel_init();
 996                 break;
 997         case X86_VENDOR_AMD:
 998                 pmc_ops = pmc_amd_init();
 999                 break;
1000         }
1001         if (!pmc_ops)
1002                 return;
1003
1004         pr_info("... num counters:    %d\n", nr_counters_generic);
1005         if (nr_counters_generic > X86_PMC_MAX_GENERIC) {
1006                 nr_counters_generic = X86_PMC_MAX_GENERIC;
1007                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1008                         nr_counters_generic, X86_PMC_MAX_GENERIC);
1009         }
1010         perf_counter_mask = (1 << nr_counters_generic) - 1;
1011         perf_max_counters = nr_counters_generic;
1012
1013         pr_info("... value mask:      %016Lx\n", counter_value_mask);
1014
1015         if (nr_counters_fixed > X86_PMC_MAX_FIXED) {
1016                 nr_counters_fixed = X86_PMC_MAX_FIXED;
1017                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1018                         nr_counters_fixed, X86_PMC_MAX_FIXED);
1019         }
1020         pr_info("... fixed counters:  %d\n", nr_counters_fixed);
1021
1022         perf_counter_mask |= ((1LL << nr_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1023
1024         pr_info("... counter mask:    %016Lx\n", perf_counter_mask);
1025         perf_counters_initialized = true;
1026
1027         perf_counters_lapic_init(0);
1028         register_die_notifier(&perf_counter_nmi_notifier);
1029 }
1030
1031 static void pmc_generic_read(struct perf_counter *counter)
1032 {
1033         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1034 }
1035
1036 static const struct hw_perf_counter_ops x86_perf_counter_ops = {
1037         .enable         = pmc_generic_enable,
1038         .disable        = pmc_generic_disable,
1039         .read           = pmc_generic_read,
1040 };
1041
1042 const struct hw_perf_counter_ops *
1043 hw_perf_counter_init(struct perf_counter *counter)
1044 {
1045         int err;
1046
1047         err = __hw_perf_counter_init(counter);
1048         if (err)
1049                 return ERR_PTR(err);
1050
1051         return &x86_perf_counter_ops;
1052 }
1053
1054 /*
1055  * callchain support
1056  */
1057
1058 static inline
1059 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1060 {
1061         if (entry->nr < MAX_STACK_DEPTH)
1062                 entry->ip[entry->nr++] = ip;
1063 }
1064
1065 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1066 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1067
1068
1069 static void
1070 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1071 {
1072         /* Ignore warnings */
1073 }
1074
1075 static void backtrace_warning(void *data, char *msg)
1076 {
1077         /* Ignore warnings */
1078 }
1079
1080 static int backtrace_stack(void *data, char *name)
1081 {
1082         /* Don't bother with IRQ stacks for now */
1083         return -1;
1084 }
1085
1086 static void backtrace_address(void *data, unsigned long addr, int reliable)
1087 {
1088         struct perf_callchain_entry *entry = data;
1089
1090         if (reliable)
1091                 callchain_store(entry, addr);
1092 }
1093
1094 static const struct stacktrace_ops backtrace_ops = {
1095         .warning                = backtrace_warning,
1096         .warning_symbol         = backtrace_warning_symbol,
1097         .stack                  = backtrace_stack,
1098         .address                = backtrace_address,
1099 };
1100
1101 static void
1102 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1103 {
1104         unsigned long bp;
1105         char *stack;
1106         int nr = entry->nr;
1107
1108         callchain_store(entry, instruction_pointer(regs));
1109
1110         stack = ((char *)regs + sizeof(struct pt_regs));
1111 #ifdef CONFIG_FRAME_POINTER
1112         bp = frame_pointer(regs);
1113 #else
1114         bp = 0;
1115 #endif
1116
1117         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1118
1119         entry->kernel = entry->nr - nr;
1120 }
1121
1122
1123 struct stack_frame {
1124         const void __user       *next_fp;
1125         unsigned long           return_address;
1126 };
1127
1128 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1129 {
1130         int ret;
1131
1132         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1133                 return 0;
1134
1135         ret = 1;
1136         pagefault_disable();
1137         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1138                 ret = 0;
1139         pagefault_enable();
1140
1141         return ret;
1142 }
1143
1144 static void
1145 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1146 {
1147         struct stack_frame frame;
1148         const void __user *fp;
1149         int nr = entry->nr;
1150
1151         regs = (struct pt_regs *)current->thread.sp0 - 1;
1152         fp   = (void __user *)regs->bp;
1153
1154         callchain_store(entry, regs->ip);
1155
1156         while (entry->nr < MAX_STACK_DEPTH) {
1157                 frame.next_fp        = NULL;
1158                 frame.return_address = 0;
1159
1160                 if (!copy_stack_frame(fp, &frame))
1161                         break;
1162
1163                 if ((unsigned long)fp < user_stack_pointer(regs))
1164                         break;
1165
1166                 callchain_store(entry, frame.return_address);
1167                 fp = frame.next_fp;
1168         }
1169
1170         entry->user = entry->nr - nr;
1171 }
1172
1173 static void
1174 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1175 {
1176         int is_user;
1177
1178         if (!regs)
1179                 return;
1180
1181         is_user = user_mode(regs);
1182
1183         if (!current || current->pid == 0)
1184                 return;
1185
1186         if (is_user && current->state != TASK_RUNNING)
1187                 return;
1188
1189         if (!is_user)
1190                 perf_callchain_kernel(regs, entry);
1191
1192         if (current->mm)
1193                 perf_callchain_user(regs, entry);
1194 }
1195
1196 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1197 {
1198         struct perf_callchain_entry *entry;
1199
1200         if (in_nmi())
1201                 entry = &__get_cpu_var(nmi_entry);
1202         else
1203                 entry = &__get_cpu_var(irq_entry);
1204
1205         entry->nr = 0;
1206         entry->hv = 0;
1207         entry->kernel = 0;
1208         entry->user = 0;
1209
1210         perf_do_callchain(regs, entry);
1211
1212         return entry;
1213 }