arch/x86/kernel/cpu/perf_event.c

   1 /*
   2  * Performance events x86 architecture code
   3  *
   4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
   6  *  Copyright (C) 2009 Jaswinder Singh Rajput
   7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
   8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
   9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
  10  *
  11  *  For licencing details see kernel-base/COPYING
  12  */
  13
  14 #include <linux/perf_event.h>
  15 #include <linux/capability.h>
  16 #include <linux/notifier.h>
  17 #include <linux/hardirq.h>
  18 #include <linux/kprobes.h>
  19 #include <linux/module.h>
  20 #include <linux/kdebug.h>
  21 #include <linux/sched.h>
  22 #include <linux/uaccess.h>
  23 #include <linux/highmem.h>
  24 #include <linux/cpu.h>
  25
  26 #include <asm/apic.h>
  27 #include <asm/stacktrace.h>
  28 #include <asm/nmi.h>
  29
  30 static u64 perf_event_mask __read_mostly;
  31
  32 /* The maximal number of PEBS events: */
  33 #define MAX_PEBS_EVENTS 4
  34
  35 /* The size of a BTS record in bytes: */
  36 #define BTS_RECORD_SIZE         24
  37
  38 /* The size of a per-cpu BTS buffer in bytes: */
  39 #define BTS_BUFFER_SIZE         (BTS_RECORD_SIZE * 2048)
  40
  41 /* The BTS overflow threshold in bytes from the end of the buffer: */
  42 #define BTS_OVFL_TH             (BTS_RECORD_SIZE * 128)
  43
  44
  45 /*
  46  * Bits in the debugctlmsr controlling branch tracing.
  47  */
  48 #define X86_DEBUGCTL_TR                 (1 << 6)
  49 #define X86_DEBUGCTL_BTS                (1 << 7)
  50 #define X86_DEBUGCTL_BTINT              (1 << 8)
  51 #define X86_DEBUGCTL_BTS_OFF_OS         (1 << 9)
  52 #define X86_DEBUGCTL_BTS_OFF_USR        (1 << 10)
  53
  54 /*
  55  * A debug store configuration.
  56  *
  57  * We only support architectures that use 64bit fields.
  58  */
  59 struct debug_store {
  60         u64     bts_buffer_base;
  61         u64     bts_index;
  62         u64     bts_absolute_maximum;
  63         u64     bts_interrupt_threshold;
  64         u64     pebs_buffer_base;
  65         u64     pebs_index;
  66         u64     pebs_absolute_maximum;
  67         u64     pebs_interrupt_threshold;
  68         u64     pebs_event_reset[MAX_PEBS_EVENTS];
  69 };
  70
  71 struct cpu_hw_events {
  72         struct perf_event       *events[X86_PMC_IDX_MAX];
  73         unsigned long           used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  74         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  75         unsigned long           interrupts;
  76         int                     enabled;
  77         struct debug_store      *ds;
  78 };
  79
  80 struct event_constraint {
  81         unsigned long   idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  82         int             code;
  83 };
  84
  85 #define EVENT_CONSTRAINT(c, m) { .code = (c), .idxmsk[0] = (m) }
  86 #define EVENT_CONSTRAINT_END  { .code = 0, .idxmsk[0] = 0 }
  87
  88 #define for_each_event_constraint(e, c) \
  89         for ((e) = (c); (e)->idxmsk[0]; (e)++)
  90
  91
  92 /*
  93  * struct x86_pmu - generic x86 pmu
  94  */
  95 struct x86_pmu {
  96         const char      *name;
  97         int             version;
  98         int             (*handle_irq)(struct pt_regs *);
  99         void            (*disable_all)(void);
 100         void            (*enable_all)(void);
 101         void            (*enable)(struct hw_perf_event *, int);
 102         void            (*disable)(struct hw_perf_event *, int);
 103         unsigned        eventsel;
 104         unsigned        perfctr;
 105         u64             (*event_map)(int);
 106         u64             (*raw_event)(u64);
 107         int             max_events;
 108         int             num_events;
 109         int             num_events_fixed;
 110         int             event_bits;
 111         u64             event_mask;
 112         int             apic;
 113         u64             max_period;
 114         u64             intel_ctrl;
 115         void            (*enable_bts)(u64 config);
 116         void            (*disable_bts)(void);
 117         int             (*get_event_idx)(struct cpu_hw_events *cpuc,
 118                                          struct hw_perf_event *hwc);
 119 };
 120
 121 static struct x86_pmu x86_pmu __read_mostly;
 122
 123 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
 124         .enabled = 1,
 125 };
 126
 127 static const struct event_constraint *event_constraints;
 128
 129 /*
 130  * Not sure about some of these
 131  */
 132 static const u64 p6_perfmon_event_map[] =
 133 {
 134   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0079,
 135   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 136   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0f2e,
 137   [PERF_COUNT_HW_CACHE_MISSES]          = 0x012e,
 138   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 139   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 140   [PERF_COUNT_HW_BUS_CYCLES]            = 0x0062,
 141 };
 142
 143 static u64 p6_pmu_event_map(int hw_event)
 144 {
 145         return p6_perfmon_event_map[hw_event];
 146 }
 147
 148 /*
 149  * Event setting that is specified not to count anything.
 150  * We use this to effectively disable a counter.
 151  *
 152  * L2_RQSTS with 0 MESI unit mask.
 153  */
 154 #define P6_NOP_EVENT                    0x0000002EULL
 155
 156 static u64 p6_pmu_raw_event(u64 hw_event)
 157 {
 158 #define P6_EVNTSEL_EVENT_MASK           0x000000FFULL
 159 #define P6_EVNTSEL_UNIT_MASK            0x0000FF00ULL
 160 #define P6_EVNTSEL_EDGE_MASK            0x00040000ULL
 161 #define P6_EVNTSEL_INV_MASK             0x00800000ULL
 162 #define P6_EVNTSEL_REG_MASK             0xFF000000ULL
 163
 164 #define P6_EVNTSEL_MASK                 \
 165         (P6_EVNTSEL_EVENT_MASK |        \
 166          P6_EVNTSEL_UNIT_MASK  |        \
 167          P6_EVNTSEL_EDGE_MASK  |        \
 168          P6_EVNTSEL_INV_MASK   |        \
 169          P6_EVNTSEL_REG_MASK)
 170
 171         return hw_event & P6_EVNTSEL_MASK;
 172 }
 173
 174 static const struct event_constraint intel_p6_event_constraints[] =
 175 {
 176         EVENT_CONSTRAINT(0xc1, 0x1),    /* FLOPS */
 177         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
 178         EVENT_CONSTRAINT(0x11, 0x1),    /* FP_ASSIST */
 179         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
 180         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
 181         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
 182         EVENT_CONSTRAINT_END
 183 };
 184
 185 /*
 186  * Intel PerfMon v3. Used on Core2 and later.
 187  */
 188 static const u64 intel_perfmon_event_map[] =
 189 {
 190   [PERF_COUNT_HW_CPU_CYCLES]            = 0x003c,
 191   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 192   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x4f2e,
 193   [PERF_COUNT_HW_CACHE_MISSES]          = 0x412e,
 194   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 195   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 196   [PERF_COUNT_HW_BUS_CYCLES]            = 0x013c,
 197 };
 198
 199 static const struct event_constraint intel_core_event_constraints[] =
 200 {
 201         EVENT_CONSTRAINT(0x10, 0x1),    /* FP_COMP_OPS_EXE */
 202         EVENT_CONSTRAINT(0x11, 0x2),    /* FP_ASSIST */
 203         EVENT_CONSTRAINT(0x12, 0x2),    /* MUL */
 204         EVENT_CONSTRAINT(0x13, 0x2),    /* DIV */
 205         EVENT_CONSTRAINT(0x14, 0x1),    /* CYCLES_DIV_BUSY */
 206         EVENT_CONSTRAINT(0x18, 0x1),    /* IDLE_DURING_DIV */
 207         EVENT_CONSTRAINT(0x19, 0x2),    /* DELAYED_BYPASS */
 208         EVENT_CONSTRAINT(0xa1, 0x1),    /* RS_UOPS_DISPATCH_CYCLES */
 209         EVENT_CONSTRAINT(0xcb, 0x1),    /* MEM_LOAD_RETIRED */
 210         EVENT_CONSTRAINT_END
 211 };
 212
 213 static const struct event_constraint intel_nehalem_event_constraints[] =
 214 {
 215         EVENT_CONSTRAINT(0x40, 0x3),    /* L1D_CACHE_LD */
 216         EVENT_CONSTRAINT(0x41, 0x3),    /* L1D_CACHE_ST */
 217         EVENT_CONSTRAINT(0x42, 0x3),    /* L1D_CACHE_LOCK */
 218         EVENT_CONSTRAINT(0x43, 0x3),    /* L1D_ALL_REF */
 219         EVENT_CONSTRAINT(0x4e, 0x3),    /* L1D_PREFETCH */
 220         EVENT_CONSTRAINT(0x4c, 0x3),    /* LOAD_HIT_PRE */
 221         EVENT_CONSTRAINT(0x51, 0x3),    /* L1D */
 222         EVENT_CONSTRAINT(0x52, 0x3),    /* L1D_CACHE_PREFETCH_LOCK_FB_HIT */
 223         EVENT_CONSTRAINT(0x53, 0x3),    /* L1D_CACHE_LOCK_FB_HIT */
 224         EVENT_CONSTRAINT(0xc5, 0x3),    /* CACHE_LOCK_CYCLES */
 225         EVENT_CONSTRAINT_END
 226 };
 227
 228 static u64 intel_pmu_event_map(int hw_event)
 229 {
 230         return intel_perfmon_event_map[hw_event];
 231 }
 232
 233 /*
 234  * Generalized hw caching related hw_event table, filled
 235  * in on a per model basis. A value of 0 means
 236  * 'not supported', -1 means 'hw_event makes no sense on
 237  * this CPU', any other value means the raw hw_event
 238  * ID.
 239  */
 240
 241 #define C(x) PERF_COUNT_HW_CACHE_##x
 242
 243 static u64 __read_mostly hw_cache_event_ids
 244                                 [PERF_COUNT_HW_CACHE_MAX]
 245                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 246                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
 247
 248 static __initconst u64 nehalem_hw_cache_event_ids
 249                                 [PERF_COUNT_HW_CACHE_MAX]
 250                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 251                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 252 {
 253  [ C(L1D) ] = {
 254         [ C(OP_READ) ] = {
 255                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI            */
 256                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE         */
 257         },
 258         [ C(OP_WRITE) ] = {
 259                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI            */
 260                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE         */
 261         },
 262         [ C(OP_PREFETCH) ] = {
 263                 [ C(RESULT_ACCESS) ] = 0x014e, /* L1D_PREFETCH.REQUESTS        */
 264                 [ C(RESULT_MISS)   ] = 0x024e, /* L1D_PREFETCH.MISS            */
 265         },
 266  },
 267  [ C(L1I ) ] = {
 268         [ C(OP_READ) ] = {
 269                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                    */
 270                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                   */
 271         },
 272         [ C(OP_WRITE) ] = {
 273                 [ C(RESULT_ACCESS) ] = -1,
 274                 [ C(RESULT_MISS)   ] = -1,
 275         },
 276         [ C(OP_PREFETCH) ] = {
 277                 [ C(RESULT_ACCESS) ] = 0x0,
 278                 [ C(RESULT_MISS)   ] = 0x0,
 279         },
 280  },
 281  [ C(LL  ) ] = {
 282         [ C(OP_READ) ] = {
 283                 [ C(RESULT_ACCESS) ] = 0x0324, /* L2_RQSTS.LOADS               */
 284                 [ C(RESULT_MISS)   ] = 0x0224, /* L2_RQSTS.LD_MISS             */
 285         },
 286         [ C(OP_WRITE) ] = {
 287                 [ C(RESULT_ACCESS) ] = 0x0c24, /* L2_RQSTS.RFOS                */
 288                 [ C(RESULT_MISS)   ] = 0x0824, /* L2_RQSTS.RFO_MISS            */
 289         },
 290         [ C(OP_PREFETCH) ] = {
 291                 [ C(RESULT_ACCESS) ] = 0x4f2e, /* LLC Reference                */
 292                 [ C(RESULT_MISS)   ] = 0x412e, /* LLC Misses                   */
 293         },
 294  },
 295  [ C(DTLB) ] = {
 296         [ C(OP_READ) ] = {
 297                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI   (alias)  */
 298                 [ C(RESULT_MISS)   ] = 0x0108, /* DTLB_LOAD_MISSES.ANY         */
 299         },
 300         [ C(OP_WRITE) ] = {
 301                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI   (alias)  */
 302                 [ C(RESULT_MISS)   ] = 0x010c, /* MEM_STORE_RETIRED.DTLB_MISS  */
 303         },
 304         [ C(OP_PREFETCH) ] = {
 305                 [ C(RESULT_ACCESS) ] = 0x0,
 306                 [ C(RESULT_MISS)   ] = 0x0,
 307         },
 308  },
 309  [ C(ITLB) ] = {
 310         [ C(OP_READ) ] = {
 311                 [ C(RESULT_ACCESS) ] = 0x01c0, /* INST_RETIRED.ANY_P           */
 312                 [ C(RESULT_MISS)   ] = 0x20c8, /* ITLB_MISS_RETIRED            */
 313         },
 314         [ C(OP_WRITE) ] = {
 315                 [ C(RESULT_ACCESS) ] = -1,
 316                 [ C(RESULT_MISS)   ] = -1,
 317         },
 318         [ C(OP_PREFETCH) ] = {
 319                 [ C(RESULT_ACCESS) ] = -1,
 320                 [ C(RESULT_MISS)   ] = -1,
 321         },
 322  },
 323  [ C(BPU ) ] = {
 324         [ C(OP_READ) ] = {
 325                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ALL_BRANCHES */
 326                 [ C(RESULT_MISS)   ] = 0x03e8, /* BPU_CLEARS.ANY               */
 327         },
 328         [ C(OP_WRITE) ] = {
 329                 [ C(RESULT_ACCESS) ] = -1,
 330                 [ C(RESULT_MISS)   ] = -1,
 331         },
 332         [ C(OP_PREFETCH) ] = {
 333                 [ C(RESULT_ACCESS) ] = -1,
 334                 [ C(RESULT_MISS)   ] = -1,
 335         },
 336  },
 337 };
 338
 339 static __initconst u64 core2_hw_cache_event_ids
 340                                 [PERF_COUNT_HW_CACHE_MAX]
 341                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 342                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 343 {
 344  [ C(L1D) ] = {
 345         [ C(OP_READ) ] = {
 346                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI          */
 347                 [ C(RESULT_MISS)   ] = 0x0140, /* L1D_CACHE_LD.I_STATE       */
 348         },
 349         [ C(OP_WRITE) ] = {
 350                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI          */
 351                 [ C(RESULT_MISS)   ] = 0x0141, /* L1D_CACHE_ST.I_STATE       */
 352         },
 353         [ C(OP_PREFETCH) ] = {
 354                 [ C(RESULT_ACCESS) ] = 0x104e, /* L1D_PREFETCH.REQUESTS      */
 355                 [ C(RESULT_MISS)   ] = 0,
 356         },
 357  },
 358  [ C(L1I ) ] = {
 359         [ C(OP_READ) ] = {
 360                 [ C(RESULT_ACCESS) ] = 0x0080, /* L1I.READS                  */
 361                 [ C(RESULT_MISS)   ] = 0x0081, /* L1I.MISSES                 */
 362         },
 363         [ C(OP_WRITE) ] = {
 364                 [ C(RESULT_ACCESS) ] = -1,
 365                 [ C(RESULT_MISS)   ] = -1,
 366         },
 367         [ C(OP_PREFETCH) ] = {
 368                 [ C(RESULT_ACCESS) ] = 0,
 369                 [ C(RESULT_MISS)   ] = 0,
 370         },
 371  },
 372  [ C(LL  ) ] = {
 373         [ C(OP_READ) ] = {
 374                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 375                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 376         },
 377         [ C(OP_WRITE) ] = {
 378                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 379                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 380         },
 381         [ C(OP_PREFETCH) ] = {
 382                 [ C(RESULT_ACCESS) ] = 0,
 383                 [ C(RESULT_MISS)   ] = 0,
 384         },
 385  },
 386  [ C(DTLB) ] = {
 387         [ C(OP_READ) ] = {
 388                 [ C(RESULT_ACCESS) ] = 0x0f40, /* L1D_CACHE_LD.MESI  (alias) */
 389                 [ C(RESULT_MISS)   ] = 0x0208, /* DTLB_MISSES.MISS_LD        */
 390         },
 391         [ C(OP_WRITE) ] = {
 392                 [ C(RESULT_ACCESS) ] = 0x0f41, /* L1D_CACHE_ST.MESI  (alias) */
 393                 [ C(RESULT_MISS)   ] = 0x0808, /* DTLB_MISSES.MISS_ST        */
 394         },
 395         [ C(OP_PREFETCH) ] = {
 396                 [ C(RESULT_ACCESS) ] = 0,
 397                 [ C(RESULT_MISS)   ] = 0,
 398         },
 399  },
 400  [ C(ITLB) ] = {
 401         [ C(OP_READ) ] = {
 402                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 403                 [ C(RESULT_MISS)   ] = 0x1282, /* ITLBMISSES                 */
 404         },
 405         [ C(OP_WRITE) ] = {
 406                 [ C(RESULT_ACCESS) ] = -1,
 407                 [ C(RESULT_MISS)   ] = -1,
 408         },
 409         [ C(OP_PREFETCH) ] = {
 410                 [ C(RESULT_ACCESS) ] = -1,
 411                 [ C(RESULT_MISS)   ] = -1,
 412         },
 413  },
 414  [ C(BPU ) ] = {
 415         [ C(OP_READ) ] = {
 416                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 417                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 418         },
 419         [ C(OP_WRITE) ] = {
 420                 [ C(RESULT_ACCESS) ] = -1,
 421                 [ C(RESULT_MISS)   ] = -1,
 422         },
 423         [ C(OP_PREFETCH) ] = {
 424                 [ C(RESULT_ACCESS) ] = -1,
 425                 [ C(RESULT_MISS)   ] = -1,
 426         },
 427  },
 428 };
 429
 430 static __initconst u64 atom_hw_cache_event_ids
 431                                 [PERF_COUNT_HW_CACHE_MAX]
 432                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 433                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 434 {
 435  [ C(L1D) ] = {
 436         [ C(OP_READ) ] = {
 437                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE.LD               */
 438                 [ C(RESULT_MISS)   ] = 0,
 439         },
 440         [ C(OP_WRITE) ] = {
 441                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE.ST               */
 442                 [ C(RESULT_MISS)   ] = 0,
 443         },
 444         [ C(OP_PREFETCH) ] = {
 445                 [ C(RESULT_ACCESS) ] = 0x0,
 446                 [ C(RESULT_MISS)   ] = 0,
 447         },
 448  },
 449  [ C(L1I ) ] = {
 450         [ C(OP_READ) ] = {
 451                 [ C(RESULT_ACCESS) ] = 0x0380, /* L1I.READS                  */
 452                 [ C(RESULT_MISS)   ] = 0x0280, /* L1I.MISSES                 */
 453         },
 454         [ C(OP_WRITE) ] = {
 455                 [ C(RESULT_ACCESS) ] = -1,
 456                 [ C(RESULT_MISS)   ] = -1,
 457         },
 458         [ C(OP_PREFETCH) ] = {
 459                 [ C(RESULT_ACCESS) ] = 0,
 460                 [ C(RESULT_MISS)   ] = 0,
 461         },
 462  },
 463  [ C(LL  ) ] = {
 464         [ C(OP_READ) ] = {
 465                 [ C(RESULT_ACCESS) ] = 0x4f29, /* L2_LD.MESI                 */
 466                 [ C(RESULT_MISS)   ] = 0x4129, /* L2_LD.ISTATE               */
 467         },
 468         [ C(OP_WRITE) ] = {
 469                 [ C(RESULT_ACCESS) ] = 0x4f2A, /* L2_ST.MESI                 */
 470                 [ C(RESULT_MISS)   ] = 0x412A, /* L2_ST.ISTATE               */
 471         },
 472         [ C(OP_PREFETCH) ] = {
 473                 [ C(RESULT_ACCESS) ] = 0,
 474                 [ C(RESULT_MISS)   ] = 0,
 475         },
 476  },
 477  [ C(DTLB) ] = {
 478         [ C(OP_READ) ] = {
 479                 [ C(RESULT_ACCESS) ] = 0x2140, /* L1D_CACHE_LD.MESI  (alias) */
 480                 [ C(RESULT_MISS)   ] = 0x0508, /* DTLB_MISSES.MISS_LD        */
 481         },
 482         [ C(OP_WRITE) ] = {
 483                 [ C(RESULT_ACCESS) ] = 0x2240, /* L1D_CACHE_ST.MESI  (alias) */
 484                 [ C(RESULT_MISS)   ] = 0x0608, /* DTLB_MISSES.MISS_ST        */
 485         },
 486         [ C(OP_PREFETCH) ] = {
 487                 [ C(RESULT_ACCESS) ] = 0,
 488                 [ C(RESULT_MISS)   ] = 0,
 489         },
 490  },
 491  [ C(ITLB) ] = {
 492         [ C(OP_READ) ] = {
 493                 [ C(RESULT_ACCESS) ] = 0x00c0, /* INST_RETIRED.ANY_P         */
 494                 [ C(RESULT_MISS)   ] = 0x0282, /* ITLB.MISSES                */
 495         },
 496         [ C(OP_WRITE) ] = {
 497                 [ C(RESULT_ACCESS) ] = -1,
 498                 [ C(RESULT_MISS)   ] = -1,
 499         },
 500         [ C(OP_PREFETCH) ] = {
 501                 [ C(RESULT_ACCESS) ] = -1,
 502                 [ C(RESULT_MISS)   ] = -1,
 503         },
 504  },
 505  [ C(BPU ) ] = {
 506         [ C(OP_READ) ] = {
 507                 [ C(RESULT_ACCESS) ] = 0x00c4, /* BR_INST_RETIRED.ANY        */
 508                 [ C(RESULT_MISS)   ] = 0x00c5, /* BP_INST_RETIRED.MISPRED    */
 509         },
 510         [ C(OP_WRITE) ] = {
 511                 [ C(RESULT_ACCESS) ] = -1,
 512                 [ C(RESULT_MISS)   ] = -1,
 513         },
 514         [ C(OP_PREFETCH) ] = {
 515                 [ C(RESULT_ACCESS) ] = -1,
 516                 [ C(RESULT_MISS)   ] = -1,
 517         },
 518  },
 519 };
 520
 521 static u64 intel_pmu_raw_event(u64 hw_event)
 522 {
 523 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
 524 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
 525 #define CORE_EVNTSEL_EDGE_MASK          0x00040000ULL
 526 #define CORE_EVNTSEL_INV_MASK           0x00800000ULL
 527 #define CORE_EVNTSEL_REG_MASK           0xFF000000ULL
 528
 529 #define CORE_EVNTSEL_MASK               \
 530         (CORE_EVNTSEL_EVENT_MASK |      \
 531          CORE_EVNTSEL_UNIT_MASK  |      \
 532          CORE_EVNTSEL_EDGE_MASK  |      \
 533          CORE_EVNTSEL_INV_MASK  |       \
 534          CORE_EVNTSEL_REG_MASK)
 535
 536         return hw_event & CORE_EVNTSEL_MASK;
 537 }
 538
 539 static __initconst u64 amd_hw_cache_event_ids
 540                                 [PERF_COUNT_HW_CACHE_MAX]
 541                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 542                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
 543 {
 544  [ C(L1D) ] = {
 545         [ C(OP_READ) ] = {
 546                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
 547                 [ C(RESULT_MISS)   ] = 0x0041, /* Data Cache Misses          */
 548         },
 549         [ C(OP_WRITE) ] = {
 550                 [ C(RESULT_ACCESS) ] = 0x0142, /* Data Cache Refills :system */
 551                 [ C(RESULT_MISS)   ] = 0,
 552         },
 553         [ C(OP_PREFETCH) ] = {
 554                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
 555                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
 556         },
 557  },
 558  [ C(L1I ) ] = {
 559         [ C(OP_READ) ] = {
 560                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
 561                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
 562         },
 563         [ C(OP_WRITE) ] = {
 564                 [ C(RESULT_ACCESS) ] = -1,
 565                 [ C(RESULT_MISS)   ] = -1,
 566         },
 567         [ C(OP_PREFETCH) ] = {
 568                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
 569                 [ C(RESULT_MISS)   ] = 0,
 570         },
 571  },
 572  [ C(LL  ) ] = {
 573         [ C(OP_READ) ] = {
 574                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
 575                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
 576         },
 577         [ C(OP_WRITE) ] = {
 578                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
 579                 [ C(RESULT_MISS)   ] = 0,
 580         },
 581         [ C(OP_PREFETCH) ] = {
 582                 [ C(RESULT_ACCESS) ] = 0,
 583                 [ C(RESULT_MISS)   ] = 0,
 584         },
 585  },
 586  [ C(DTLB) ] = {
 587         [ C(OP_READ) ] = {
 588                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
 589                 [ C(RESULT_MISS)   ] = 0x0046, /* L1 DTLB and L2 DLTB Miss   */
 590         },
 591         [ C(OP_WRITE) ] = {
 592                 [ C(RESULT_ACCESS) ] = 0,
 593                 [ C(RESULT_MISS)   ] = 0,
 594         },
 595         [ C(OP_PREFETCH) ] = {
 596                 [ C(RESULT_ACCESS) ] = 0,
 597                 [ C(RESULT_MISS)   ] = 0,
 598         },
 599  },
 600  [ C(ITLB) ] = {
 601         [ C(OP_READ) ] = {
 602                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
 603                 [ C(RESULT_MISS)   ] = 0x0085, /* Instr. fetch ITLB misses   */
 604         },
 605         [ C(OP_WRITE) ] = {
 606                 [ C(RESULT_ACCESS) ] = -1,
 607                 [ C(RESULT_MISS)   ] = -1,
 608         },
 609         [ C(OP_PREFETCH) ] = {
 610                 [ C(RESULT_ACCESS) ] = -1,
 611                 [ C(RESULT_MISS)   ] = -1,
 612         },
 613  },
 614  [ C(BPU ) ] = {
 615         [ C(OP_READ) ] = {
 616                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 617                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 618         },
 619         [ C(OP_WRITE) ] = {
 620                 [ C(RESULT_ACCESS) ] = -1,
 621                 [ C(RESULT_MISS)   ] = -1,
 622         },
 623         [ C(OP_PREFETCH) ] = {
 624                 [ C(RESULT_ACCESS) ] = -1,
 625                 [ C(RESULT_MISS)   ] = -1,
 626         },
 627  },
 628 };
 629
 630 /*
 631  * AMD Performance Monitor K7 and later.
 632  */
 633 static const u64 amd_perfmon_event_map[] =
 634 {
 635   [PERF_COUNT_HW_CPU_CYCLES]            = 0x0076,
 636   [PERF_COUNT_HW_INSTRUCTIONS]          = 0x00c0,
 637   [PERF_COUNT_HW_CACHE_REFERENCES]      = 0x0080,
 638   [PERF_COUNT_HW_CACHE_MISSES]          = 0x0081,
 639   [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]   = 0x00c4,
 640   [PERF_COUNT_HW_BRANCH_MISSES]         = 0x00c5,
 641 };
 642
 643 static u64 amd_pmu_event_map(int hw_event)
 644 {
 645         return amd_perfmon_event_map[hw_event];
 646 }
 647
 648 static u64 amd_pmu_raw_event(u64 hw_event)
 649 {
 650 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 651 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 652 #define K7_EVNTSEL_EDGE_MASK    0x000040000ULL
 653 #define K7_EVNTSEL_INV_MASK     0x000800000ULL
 654 #define K7_EVNTSEL_REG_MASK     0x0FF000000ULL
 655
 656 #define K7_EVNTSEL_MASK                 \
 657         (K7_EVNTSEL_EVENT_MASK |        \
 658          K7_EVNTSEL_UNIT_MASK  |        \
 659          K7_EVNTSEL_EDGE_MASK  |        \
 660          K7_EVNTSEL_INV_MASK   |        \
 661          K7_EVNTSEL_REG_MASK)
 662
 663         return hw_event & K7_EVNTSEL_MASK;
 664 }
 665
 666 /*
 667  * Propagate event elapsed time into the generic event.
 668  * Can only be executed on the CPU where the event is active.
 669  * Returns the delta events processed.
 670  */
 671 static u64
 672 x86_perf_event_update(struct perf_event *event,
 673                         struct hw_perf_event *hwc, int idx)
 674 {
 675         int shift = 64 - x86_pmu.event_bits;
 676         u64 prev_raw_count, new_raw_count;
 677         s64 delta;
 678
 679         if (idx == X86_PMC_IDX_FIXED_BTS)
 680                 return 0;
 681
 682         /*
 683          * Careful: an NMI might modify the previous event value.
 684          *
 685          * Our tactic to handle this is to first atomically read and
 686          * exchange a new raw count - then add that new-prev delta
 687          * count to the generic event atomically:
 688          */
 689 again:
 690         prev_raw_count = atomic64_read(&hwc->prev_count);
 691         rdmsrl(hwc->event_base + idx, new_raw_count);
 692
 693         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 694                                         new_raw_count) != prev_raw_count)
 695                 goto again;
 696
 697         /*
 698          * Now we have the new raw value and have updated the prev
 699          * timestamp already. We can now calculate the elapsed delta
 700          * (event-)time and add that to the generic event.
 701          *
 702          * Careful, not all hw sign-extends above the physical width
 703          * of the count.
 704          */
 705         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 706         delta >>= shift;
 707
 708         atomic64_add(delta, &event->count);
 709         atomic64_sub(delta, &hwc->period_left);
 710
 711         return new_raw_count;
 712 }
 713
 714 static atomic_t active_events;
 715 static DEFINE_MUTEX(pmc_reserve_mutex);
 716
 717 static bool reserve_pmc_hardware(void)
 718 {
 719 #ifdef CONFIG_X86_LOCAL_APIC
 720         int i;
 721
 722         if (nmi_watchdog == NMI_LOCAL_APIC)
 723                 disable_lapic_nmi_watchdog();
 724
 725         for (i = 0; i < x86_pmu.num_events; i++) {
 726                 if (!reserve_perfctr_nmi(x86_pmu.perfctr + i))
 727                         goto perfctr_fail;
 728         }
 729
 730         for (i = 0; i < x86_pmu.num_events; i++) {
 731                 if (!reserve_evntsel_nmi(x86_pmu.eventsel + i))
 732                         goto eventsel_fail;
 733         }
 734 #endif
 735
 736         return true;
 737
 738 #ifdef CONFIG_X86_LOCAL_APIC
 739 eventsel_fail:
 740         for (i--; i >= 0; i--)
 741                 release_evntsel_nmi(x86_pmu.eventsel + i);
 742
 743         i = x86_pmu.num_events;
 744
 745 perfctr_fail:
 746         for (i--; i >= 0; i--)
 747                 release_perfctr_nmi(x86_pmu.perfctr + i);
 748
 749         if (nmi_watchdog == NMI_LOCAL_APIC)
 750                 enable_lapic_nmi_watchdog();
 751
 752         return false;
 753 #endif
 754 }
 755
 756 static void release_pmc_hardware(void)
 757 {
 758 #ifdef CONFIG_X86_LOCAL_APIC
 759         int i;
 760
 761         for (i = 0; i < x86_pmu.num_events; i++) {
 762                 release_perfctr_nmi(x86_pmu.perfctr + i);
 763                 release_evntsel_nmi(x86_pmu.eventsel + i);
 764         }
 765
 766         if (nmi_watchdog == NMI_LOCAL_APIC)
 767                 enable_lapic_nmi_watchdog();
 768 #endif
 769 }
 770
 771 static inline bool bts_available(void)
 772 {
 773         return x86_pmu.enable_bts != NULL;
 774 }
 775
 776 static inline void init_debug_store_on_cpu(int cpu)
 777 {
 778         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 779
 780         if (!ds)
 781                 return;
 782
 783         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
 784                      (u32)((u64)(unsigned long)ds),
 785                      (u32)((u64)(unsigned long)ds >> 32));
 786 }
 787
 788 static inline void fini_debug_store_on_cpu(int cpu)
 789 {
 790         if (!per_cpu(cpu_hw_events, cpu).ds)
 791                 return;
 792
 793         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
 794 }
 795
 796 static void release_bts_hardware(void)
 797 {
 798         int cpu;
 799
 800         if (!bts_available())
 801                 return;
 802
 803         get_online_cpus();
 804
 805         for_each_online_cpu(cpu)
 806                 fini_debug_store_on_cpu(cpu);
 807
 808         for_each_possible_cpu(cpu) {
 809                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
 810
 811                 if (!ds)
 812                         continue;
 813
 814                 per_cpu(cpu_hw_events, cpu).ds = NULL;
 815
 816                 kfree((void *)(unsigned long)ds->bts_buffer_base);
 817                 kfree(ds);
 818         }
 819
 820         put_online_cpus();
 821 }
 822
 823 static int reserve_bts_hardware(void)
 824 {
 825         int cpu, err = 0;
 826
 827         if (!bts_available())
 828                 return 0;
 829
 830         get_online_cpus();
 831
 832         for_each_possible_cpu(cpu) {
 833                 struct debug_store *ds;
 834                 void *buffer;
 835
 836                 err = -ENOMEM;
 837                 buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
 838                 if (unlikely(!buffer))
 839                         break;
 840
 841                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
 842                 if (unlikely(!ds)) {
 843                         kfree(buffer);
 844                         break;
 845                 }
 846
 847                 ds->bts_buffer_base = (u64)(unsigned long)buffer;
 848                 ds->bts_index = ds->bts_buffer_base;
 849                 ds->bts_absolute_maximum =
 850                         ds->bts_buffer_base + BTS_BUFFER_SIZE;
 851                 ds->bts_interrupt_threshold =
 852                         ds->bts_absolute_maximum - BTS_OVFL_TH;
 853
 854                 per_cpu(cpu_hw_events, cpu).ds = ds;
 855                 err = 0;
 856         }
 857
 858         if (err)
 859                 release_bts_hardware();
 860         else {
 861                 for_each_online_cpu(cpu)
 862                         init_debug_store_on_cpu(cpu);
 863         }
 864
 865         put_online_cpus();
 866
 867         return err;
 868 }
 869
 870 static void hw_perf_event_destroy(struct perf_event *event)
 871 {
 872         if (atomic_dec_and_mutex_lock(&active_events, &pmc_reserve_mutex)) {
 873                 release_pmc_hardware();
 874                 release_bts_hardware();
 875                 mutex_unlock(&pmc_reserve_mutex);
 876         }
 877 }
 878
 879 static inline int x86_pmu_initialized(void)
 880 {
 881         return x86_pmu.handle_irq != NULL;
 882 }
 883
 884 static inline int
 885 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event_attr *attr)
 886 {
 887         unsigned int cache_type, cache_op, cache_result;
 888         u64 config, val;
 889
 890         config = attr->config;
 891
 892         cache_type = (config >>  0) & 0xff;
 893         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 894                 return -EINVAL;
 895
 896         cache_op = (config >>  8) & 0xff;
 897         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 898                 return -EINVAL;
 899
 900         cache_result = (config >> 16) & 0xff;
 901         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 902                 return -EINVAL;
 903
 904         val = hw_cache_event_ids[cache_type][cache_op][cache_result];
 905
 906         if (val == 0)
 907                 return -ENOENT;
 908
 909         if (val == -1)
 910                 return -EINVAL;
 911
 912         hwc->config |= val;
 913
 914         return 0;
 915 }
 916
 917 static void intel_pmu_enable_bts(u64 config)
 918 {
 919         unsigned long debugctlmsr;
 920
 921         debugctlmsr = get_debugctlmsr();
 922
 923         debugctlmsr |= X86_DEBUGCTL_TR;
 924         debugctlmsr |= X86_DEBUGCTL_BTS;
 925         debugctlmsr |= X86_DEBUGCTL_BTINT;
 926
 927         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
 928                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;
 929
 930         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
 931                 debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;
 932
 933         update_debugctlmsr(debugctlmsr);
 934 }
 935
 936 static void intel_pmu_disable_bts(void)
 937 {
 938         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 939         unsigned long debugctlmsr;
 940
 941         if (!cpuc->ds)
 942                 return;
 943
 944         debugctlmsr = get_debugctlmsr();
 945
 946         debugctlmsr &=
 947                 ~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
 948                   X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);
 949
 950         update_debugctlmsr(debugctlmsr);
 951 }
 952
 953 /*
 954  * Setup the hardware configuration for a given attr_type
 955  */
 956 static int __hw_perf_event_init(struct perf_event *event)
 957 {
 958         struct perf_event_attr *attr = &event->attr;
 959         struct hw_perf_event *hwc = &event->hw;
 960         u64 config;
 961         int err;
 962
 963         if (!x86_pmu_initialized())
 964                 return -ENODEV;
 965
 966         err = 0;
 967         if (!atomic_inc_not_zero(&active_events)) {
 968                 mutex_lock(&pmc_reserve_mutex);
 969                 if (atomic_read(&active_events) == 0) {
 970                         if (!reserve_pmc_hardware())
 971                                 err = -EBUSY;
 972                         else
 973                                 err = reserve_bts_hardware();
 974                 }
 975                 if (!err)
 976                         atomic_inc(&active_events);
 977                 mutex_unlock(&pmc_reserve_mutex);
 978         }
 979         if (err)
 980                 return err;
 981
 982         event->destroy = hw_perf_event_destroy;
 983
 984         /*
 985          * Generate PMC IRQs:
 986          * (keep 'enabled' bit clear for now)
 987          */
 988         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 989
 990         hwc->idx = -1;
 991
 992         /*
 993          * Count user and OS events unless requested not to.
 994          */
 995         if (!attr->exclude_user)
 996                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 997         if (!attr->exclude_kernel)
 998                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 999
1000         if (!hwc->sample_period) {
1001                 hwc->sample_period = x86_pmu.max_period;
1002                 hwc->last_period = hwc->sample_period;
1003                 atomic64_set(&hwc->period_left, hwc->sample_period);
1004         } else {
1005                 /*
1006                  * If we have a PMU initialized but no APIC
1007                  * interrupts, we cannot sample hardware
1008                  * events (user-space has to fall back and
1009                  * sample via a hrtimer based software event):
1010                  */
1011                 if (!x86_pmu.apic)
1012                         return -EOPNOTSUPP;
1013         }
1014
1015         /*
1016          * Raw hw_event type provide the config in the hw_event structure
1017          */
1018         if (attr->type == PERF_TYPE_RAW) {
1019                 hwc->config |= x86_pmu.raw_event(attr->config);
1020                 return 0;
1021         }
1022
1023         if (attr->type == PERF_TYPE_HW_CACHE)
1024                 return set_ext_hw_attr(hwc, attr);
1025
1026         if (attr->config >= x86_pmu.max_events)
1027                 return -EINVAL;
1028
1029         /*
1030          * The generic map:
1031          */
1032         config = x86_pmu.event_map(attr->config);
1033
1034         if (config == 0)
1035                 return -ENOENT;
1036
1037         if (config == -1LL)
1038                 return -EINVAL;
1039
1040         /*
1041          * Branch tracing:
1042          */
1043         if ((attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS) &&
1044             (hwc->sample_period == 1)) {
1045                 /* BTS is not supported by this architecture. */
1046                 if (!bts_available())
1047                         return -EOPNOTSUPP;
1048
1049                 /* BTS is currently only allowed for user-mode. */
1050                 if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1051                         return -EOPNOTSUPP;
1052         }
1053
1054         hwc->config |= config;
1055
1056         return 0;
1057 }
1058
1059 static void p6_pmu_disable_all(void)
1060 {
1061         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1062         u64 val;
1063
1064         if (!cpuc->enabled)
1065                 return;
1066
1067         cpuc->enabled = 0;
1068         barrier();
1069
1070         /* p6 only has one enable register */
1071         rdmsrl(MSR_P6_EVNTSEL0, val);
1072         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1073         wrmsrl(MSR_P6_EVNTSEL0, val);
1074 }
1075
1076 static void intel_pmu_disable_all(void)
1077 {
1078         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1079
1080         if (!cpuc->enabled)
1081                 return;
1082
1083         cpuc->enabled = 0;
1084         barrier();
1085
1086         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
1087
1088         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1089                 intel_pmu_disable_bts();
1090 }
1091
1092 static void amd_pmu_disable_all(void)
1093 {
1094         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1095         int idx;
1096
1097         if (!cpuc->enabled)
1098                 return;
1099
1100         cpuc->enabled = 0;
1101         /*
1102          * ensure we write the disable before we start disabling the
1103          * events proper, so that amd_pmu_enable_event() does the
1104          * right thing.
1105          */
1106         barrier();
1107
1108         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1109                 u64 val;
1110
1111                 if (!test_bit(idx, cpuc->active_mask))
1112                         continue;
1113                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
1114                 if (!(val & ARCH_PERFMON_EVENTSEL0_ENABLE))
1115                         continue;
1116                 val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
1117                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1118         }
1119 }
1120
1121 void hw_perf_disable(void)
1122 {
1123         if (!x86_pmu_initialized())
1124                 return;
1125         return x86_pmu.disable_all();
1126 }
1127
1128 static void p6_pmu_enable_all(void)
1129 {
1130         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1131         unsigned long val;
1132
1133         if (cpuc->enabled)
1134                 return;
1135
1136         cpuc->enabled = 1;
1137         barrier();
1138
1139         /* p6 only has one enable register */
1140         rdmsrl(MSR_P6_EVNTSEL0, val);
1141         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1142         wrmsrl(MSR_P6_EVNTSEL0, val);
1143 }
1144
1145 static void intel_pmu_enable_all(void)
1146 {
1147         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1148
1149         if (cpuc->enabled)
1150                 return;
1151
1152         cpuc->enabled = 1;
1153         barrier();
1154
1155         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, x86_pmu.intel_ctrl);
1156
1157         if (test_bit(X86_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
1158                 struct perf_event *event =
1159                         cpuc->events[X86_PMC_IDX_FIXED_BTS];
1160
1161                 if (WARN_ON_ONCE(!event))
1162                         return;
1163
1164                 intel_pmu_enable_bts(event->hw.config);
1165         }
1166 }
1167
1168 static void amd_pmu_enable_all(void)
1169 {
1170         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1171         int idx;
1172
1173         if (cpuc->enabled)
1174                 return;
1175
1176         cpuc->enabled = 1;
1177         barrier();
1178
1179         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1180                 struct perf_event *event = cpuc->events[idx];
1181                 u64 val;
1182
1183                 if (!test_bit(idx, cpuc->active_mask))
1184                         continue;
1185
1186                 val = event->hw.config;
1187                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1188                 wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
1189         }
1190 }
1191
1192 void hw_perf_enable(void)
1193 {
1194         if (!x86_pmu_initialized())
1195                 return;
1196         x86_pmu.enable_all();
1197 }
1198
1199 static inline u64 intel_pmu_get_status(void)
1200 {
1201         u64 status;
1202
1203         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1204
1205         return status;
1206 }
1207
1208 static inline void intel_pmu_ack_status(u64 ack)
1209 {
1210         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
1211 }
1212
1213 static inline void x86_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1214 {
1215         (void)checking_wrmsrl(hwc->config_base + idx,
1216                               hwc->config | ARCH_PERFMON_EVENTSEL0_ENABLE);
1217 }
1218
1219 static inline void x86_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1220 {
1221         (void)checking_wrmsrl(hwc->config_base + idx, hwc->config);
1222 }
1223
1224 static inline void
1225 intel_pmu_disable_fixed(struct hw_perf_event *hwc, int __idx)
1226 {
1227         int idx = __idx - X86_PMC_IDX_FIXED;
1228         u64 ctrl_val, mask;
1229
1230         mask = 0xfULL << (idx * 4);
1231
1232         rdmsrl(hwc->config_base, ctrl_val);
1233         ctrl_val &= ~mask;
1234         (void)checking_wrmsrl(hwc->config_base, ctrl_val);
1235 }
1236
1237 static inline void
1238 p6_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1239 {
1240         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1241         u64 val = P6_NOP_EVENT;
1242
1243         if (cpuc->enabled)
1244                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1245
1246         (void)checking_wrmsrl(hwc->config_base + idx, val);
1247 }
1248
1249 static inline void
1250 intel_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1251 {
1252         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1253                 intel_pmu_disable_bts();
1254                 return;
1255         }
1256
1257         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1258                 intel_pmu_disable_fixed(hwc, idx);
1259                 return;
1260         }
1261
1262         x86_pmu_disable_event(hwc, idx);
1263 }
1264
1265 static inline void
1266 amd_pmu_disable_event(struct hw_perf_event *hwc, int idx)
1267 {
1268         x86_pmu_disable_event(hwc, idx);
1269 }
1270
1271 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1272
1273 /*
1274  * Set the next IRQ period, based on the hwc->period_left value.
1275  * To be called with the event disabled in hw:
1276  */
1277 static int
1278 x86_perf_event_set_period(struct perf_event *event,
1279                              struct hw_perf_event *hwc, int idx)
1280 {
1281         s64 left = atomic64_read(&hwc->period_left);
1282         s64 period = hwc->sample_period;
1283         int err, ret = 0;
1284
1285         if (idx == X86_PMC_IDX_FIXED_BTS)
1286                 return 0;
1287
1288         /*
1289          * If we are way outside a reasonable range then just skip forward:
1290          */
1291         if (unlikely(left <= -period)) {
1292                 left = period;
1293                 atomic64_set(&hwc->period_left, left);
1294                 hwc->last_period = period;
1295                 ret = 1;
1296         }
1297
1298         if (unlikely(left <= 0)) {
1299                 left += period;
1300                 atomic64_set(&hwc->period_left, left);
1301                 hwc->last_period = period;
1302                 ret = 1;
1303         }
1304         /*
1305          * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1306          */
1307         if (unlikely(left < 2))
1308                 left = 2;
1309
1310         if (left > x86_pmu.max_period)
1311                 left = x86_pmu.max_period;
1312
1313         per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1314
1315         /*
1316          * The hw event starts counting from this event offset,
1317          * mark it to be able to extra future deltas:
1318          */
1319         atomic64_set(&hwc->prev_count, (u64)-left);
1320
1321         err = checking_wrmsrl(hwc->event_base + idx,
1322                              (u64)(-left) & x86_pmu.event_mask);
1323
1324         perf_event_update_userpage(event);
1325
1326         return ret;
1327 }
1328
1329 static inline void
1330 intel_pmu_enable_fixed(struct hw_perf_event *hwc, int __idx)
1331 {
1332         int idx = __idx - X86_PMC_IDX_FIXED;
1333         u64 ctrl_val, bits, mask;
1334         int err;
1335
1336         /*
1337          * Enable IRQ generation (0x8),
1338          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
1339          * if requested:
1340          */
1341         bits = 0x8ULL;
1342         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
1343                 bits |= 0x2;
1344         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
1345                 bits |= 0x1;
1346
1347         /*
1348          * ANY bit is supported in v3 and up
1349          */
1350         if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
1351                 bits |= 0x4;
1352
1353         bits <<= (idx * 4);
1354         mask = 0xfULL << (idx * 4);
1355
1356         rdmsrl(hwc->config_base, ctrl_val);
1357         ctrl_val &= ~mask;
1358         ctrl_val |= bits;
1359         err = checking_wrmsrl(hwc->config_base, ctrl_val);
1360 }
1361
1362 static void p6_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1363 {
1364         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1365         u64 val;
1366
1367         val = hwc->config;
1368         if (cpuc->enabled)
1369                 val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
1370
1371         (void)checking_wrmsrl(hwc->config_base + idx, val);
1372 }
1373
1374
1375 static void intel_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1376 {
1377         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS)) {
1378                 if (!__get_cpu_var(cpu_hw_events).enabled)
1379                         return;
1380
1381                 intel_pmu_enable_bts(hwc->config);
1382                 return;
1383         }
1384
1385         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
1386                 intel_pmu_enable_fixed(hwc, idx);
1387                 return;
1388         }
1389
1390         x86_pmu_enable_event(hwc, idx);
1391 }
1392
1393 static void amd_pmu_enable_event(struct hw_perf_event *hwc, int idx)
1394 {
1395         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1396
1397         if (cpuc->enabled)
1398                 x86_pmu_enable_event(hwc, idx);
1399 }
1400
1401 static int fixed_mode_idx(struct hw_perf_event *hwc)
1402 {
1403         unsigned int hw_event;
1404
1405         hw_event = hwc->config & ARCH_PERFMON_EVENT_MASK;
1406
1407         if (unlikely((hw_event ==
1408                       x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS)) &&
1409                      (hwc->sample_period == 1)))
1410                 return X86_PMC_IDX_FIXED_BTS;
1411
1412         if (!x86_pmu.num_events_fixed)
1413                 return -1;
1414
1415         /*
1416          * fixed counters do not take all possible filters
1417          */
1418         if (hwc->config & ARCH_PERFMON_EVENT_FILTER_MASK)
1419                 return -1;
1420
1421         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_INSTRUCTIONS)))
1422                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
1423         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_CPU_CYCLES)))
1424                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
1425         if (unlikely(hw_event == x86_pmu.event_map(PERF_COUNT_HW_BUS_CYCLES)))
1426                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
1427
1428         return -1;
1429 }
1430
1431 /*
1432  * generic counter allocator: get next free counter
1433  */
1434 static int
1435 gen_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1436 {
1437         int idx;
1438
1439         idx = find_first_zero_bit(cpuc->used_mask, x86_pmu.num_events);
1440         return idx == x86_pmu.num_events ? -1 : idx;
1441 }
1442
1443 /*
1444  * intel-specific counter allocator: check event constraints
1445  */
1446 static int
1447 intel_get_event_idx(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1448 {
1449         const struct event_constraint *event_constraint;
1450         int i, code;
1451
1452         if (!event_constraints)
1453                 goto skip;
1454
1455         code = hwc->config & CORE_EVNTSEL_EVENT_MASK;
1456
1457         for_each_event_constraint(event_constraint, event_constraints) {
1458                 if (code == event_constraint->code) {
1459                         for_each_bit(i, event_constraint->idxmsk, X86_PMC_IDX_MAX) {
1460                                 if (!test_and_set_bit(i, cpuc->used_mask))
1461                                         return i;
1462                         }
1463                         return -1;
1464                 }
1465         }
1466 skip:
1467         return gen_get_event_idx(cpuc, hwc);
1468 }
1469
1470 static int
1471 x86_schedule_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc)
1472 {
1473         int idx;
1474
1475         idx = fixed_mode_idx(hwc);
1476         if (idx == X86_PMC_IDX_FIXED_BTS) {
1477                 /* BTS is already occupied. */
1478                 if (test_and_set_bit(idx, cpuc->used_mask))
1479                         return -EAGAIN;
1480
1481                 hwc->config_base        = 0;
1482                 hwc->event_base         = 0;
1483                 hwc->idx                = idx;
1484         } else if (idx >= 0) {
1485                 /*
1486                  * Try to get the fixed event, if that is already taken
1487                  * then try to get a generic event:
1488                  */
1489                 if (test_and_set_bit(idx, cpuc->used_mask))
1490                         goto try_generic;
1491
1492                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1493                 /*
1494                  * We set it so that event_base + idx in wrmsr/rdmsr maps to
1495                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
1496                  */
1497                 hwc->event_base =
1498                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
1499                 hwc->idx = idx;
1500         } else {
1501                 idx = hwc->idx;
1502                 /* Try to get the previous generic event again */
1503                 if (idx == -1 || test_and_set_bit(idx, cpuc->used_mask)) {
1504 try_generic:
1505                         idx = x86_pmu.get_event_idx(cpuc, hwc);
1506                         if (idx == -1)
1507                                 return -EAGAIN;
1508
1509                         set_bit(idx, cpuc->used_mask);
1510                         hwc->idx = idx;
1511                 }
1512                 hwc->config_base = x86_pmu.eventsel;
1513                 hwc->event_base  = x86_pmu.perfctr;
1514         }
1515
1516         return idx;
1517 }
1518
1519 /*
1520  * Find a PMC slot for the freshly enabled / scheduled in event:
1521  */
1522 static int x86_pmu_enable(struct perf_event *event)
1523 {
1524         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1525         struct hw_perf_event *hwc = &event->hw;
1526         int idx;
1527
1528         idx = x86_schedule_event(cpuc, hwc);
1529         if (idx < 0)
1530                 return idx;
1531
1532         perf_events_lapic_init();
1533
1534         x86_pmu.disable(hwc, idx);
1535
1536         cpuc->events[idx] = event;
1537         set_bit(idx, cpuc->active_mask);
1538
1539         x86_perf_event_set_period(event, hwc, idx);
1540         x86_pmu.enable(hwc, idx);
1541
1542         perf_event_update_userpage(event);
1543
1544         return 0;
1545 }
1546
1547 static void x86_pmu_unthrottle(struct perf_event *event)
1548 {
1549         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1550         struct hw_perf_event *hwc = &event->hw;
1551
1552         if (WARN_ON_ONCE(hwc->idx >= X86_PMC_IDX_MAX ||
1553                                 cpuc->events[hwc->idx] != event))
1554                 return;
1555
1556         x86_pmu.enable(hwc, hwc->idx);
1557 }
1558
1559 void perf_event_print_debug(void)
1560 {
1561         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1562         struct cpu_hw_events *cpuc;
1563         unsigned long flags;
1564         int cpu, idx;
1565
1566         if (!x86_pmu.num_events)
1567                 return;
1568
1569         local_irq_save(flags);
1570
1571         cpu = smp_processor_id();
1572         cpuc = &per_cpu(cpu_hw_events, cpu);
1573
1574         if (x86_pmu.version >= 2) {
1575                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1576                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1577                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1578                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1579
1580                 pr_info("\n");
1581                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1582                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1583                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1584                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1585         }
1586         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used_mask);
1587
1588         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1589                 rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
1590                 rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
1591
1592                 prev_left = per_cpu(pmc_prev_left[idx], cpu);
1593
1594                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1595                         cpu, idx, pmc_ctrl);
1596                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1597                         cpu, idx, pmc_count);
1598                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1599                         cpu, idx, prev_left);
1600         }
1601         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1602                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1603
1604                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1605                         cpu, idx, pmc_count);
1606         }
1607         local_irq_restore(flags);
1608 }
1609
1610 static void intel_pmu_drain_bts_buffer(struct cpu_hw_events *cpuc)
1611 {
1612         struct debug_store *ds = cpuc->ds;
1613         struct bts_record {
1614                 u64     from;
1615                 u64     to;
1616                 u64     flags;
1617         };
1618         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
1619         struct bts_record *at, *top;
1620         struct perf_output_handle handle;
1621         struct perf_event_header header;
1622         struct perf_sample_data data;
1623         struct pt_regs regs;
1624
1625         if (!event)
1626                 return;
1627
1628         if (!ds)
1629                 return;
1630
1631         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
1632         top = (struct bts_record *)(unsigned long)ds->bts_index;
1633
1634         if (top <= at)
1635                 return;
1636
1637         ds->bts_index = ds->bts_buffer_base;
1638
1639         perf_sample_data_init(&data, 0);
1640
1641         data.period     = event->hw.last_period;
1642         regs.ip         = 0;
1643
1644         /*
1645          * Prepare a generic sample, i.e. fill in the invariant fields.
1646          * We will overwrite the from and to address before we output
1647          * the sample.
1648          */
1649         perf_prepare_sample(&header, &data, event, &regs);
1650
1651         if (perf_output_begin(&handle, event,
1652                               header.size * (top - at), 1, 1))
1653                 return;
1654
1655         for (; at < top; at++) {
1656                 data.ip         = at->from;
1657                 data.addr       = at->to;
1658
1659                 perf_output_sample(&handle, &header, &data, event);
1660         }
1661
1662         perf_output_end(&handle);
1663
1664         /* There's new data available. */
1665         event->hw.interrupts++;
1666         event->pending_kill = POLL_IN;
1667 }
1668
1669 static void x86_pmu_disable(struct perf_event *event)
1670 {
1671         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
1672         struct hw_perf_event *hwc = &event->hw;
1673         int idx = hwc->idx;
1674
1675         /*
1676          * Must be done before we disable, otherwise the nmi handler
1677          * could reenable again:
1678          */
1679         clear_bit(idx, cpuc->active_mask);
1680         x86_pmu.disable(hwc, idx);
1681
1682         /*
1683          * Make sure the cleared pointer becomes visible before we
1684          * (potentially) free the event:
1685          */
1686         barrier();
1687
1688         /*
1689          * Drain the remaining delta count out of a event
1690          * that we are disabling:
1691          */
1692         x86_perf_event_update(event, hwc, idx);
1693
1694         /* Drain the remaining BTS records. */
1695         if (unlikely(idx == X86_PMC_IDX_FIXED_BTS))
1696                 intel_pmu_drain_bts_buffer(cpuc);
1697
1698         cpuc->events[idx] = NULL;
1699         clear_bit(idx, cpuc->used_mask);
1700
1701         perf_event_update_userpage(event);
1702 }
1703
1704 /*
1705  * Save and restart an expired event. Called by NMI contexts,
1706  * so it has to be careful about preempting normal event ops:
1707  */
1708 static int intel_pmu_save_and_restart(struct perf_event *event)
1709 {
1710         struct hw_perf_event *hwc = &event->hw;
1711         int idx = hwc->idx;
1712         int ret;
1713
1714         x86_perf_event_update(event, hwc, idx);
1715         ret = x86_perf_event_set_period(event, hwc, idx);
1716
1717         if (event->state == PERF_EVENT_STATE_ACTIVE)
1718                 intel_pmu_enable_event(hwc, idx);
1719
1720         return ret;
1721 }
1722
1723 static void intel_pmu_reset(void)
1724 {
1725         struct debug_store *ds = __get_cpu_var(cpu_hw_events).ds;
1726         unsigned long flags;
1727         int idx;
1728
1729         if (!x86_pmu.num_events)
1730                 return;
1731
1732         local_irq_save(flags);
1733
1734         printk("clearing PMU state on CPU#%d\n", smp_processor_id());
1735
1736         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1737                 checking_wrmsrl(x86_pmu.eventsel + idx, 0ull);
1738                 checking_wrmsrl(x86_pmu.perfctr  + idx, 0ull);
1739         }
1740         for (idx = 0; idx < x86_pmu.num_events_fixed; idx++) {
1741                 checking_wrmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, 0ull);
1742         }
1743         if (ds)
1744                 ds->bts_index = ds->bts_buffer_base;
1745
1746         local_irq_restore(flags);
1747 }
1748
1749 static int p6_pmu_handle_irq(struct pt_regs *regs)
1750 {
1751         struct perf_sample_data data;
1752         struct cpu_hw_events *cpuc;
1753         struct perf_event *event;
1754         struct hw_perf_event *hwc;
1755         int idx, handled = 0;
1756         u64 val;
1757
1758         perf_sample_data_init(&data, 0);
1759
1760         cpuc = &__get_cpu_var(cpu_hw_events);
1761
1762         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1763                 if (!test_bit(idx, cpuc->active_mask))
1764                         continue;
1765
1766                 event = cpuc->events[idx];
1767                 hwc = &event->hw;
1768
1769                 val = x86_perf_event_update(event, hwc, idx);
1770                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1771                         continue;
1772
1773                 /*
1774                  * event overflow
1775                  */
1776                 handled         = 1;
1777                 data.period     = event->hw.last_period;
1778
1779                 if (!x86_perf_event_set_period(event, hwc, idx))
1780                         continue;
1781
1782                 if (perf_event_overflow(event, 1, &data, regs))
1783                         p6_pmu_disable_event(hwc, idx);
1784         }
1785
1786         if (handled)
1787                 inc_irq_stat(apic_perf_irqs);
1788
1789         return handled;
1790 }
1791
1792 /*
1793  * This handler is triggered by the local APIC, so the APIC IRQ handling
1794  * rules apply:
1795  */
1796 static int intel_pmu_handle_irq(struct pt_regs *regs)
1797 {
1798         struct perf_sample_data data;
1799         struct cpu_hw_events *cpuc;
1800         int bit, loops;
1801         u64 ack, status;
1802
1803         perf_sample_data_init(&data, 0);
1804
1805         cpuc = &__get_cpu_var(cpu_hw_events);
1806
1807         perf_disable();
1808         intel_pmu_drain_bts_buffer(cpuc);
1809         status = intel_pmu_get_status();
1810         if (!status) {
1811                 perf_enable();
1812                 return 0;
1813         }
1814
1815         loops = 0;
1816 again:
1817         if (++loops > 100) {
1818                 WARN_ONCE(1, "perfevents: irq loop stuck!\n");
1819                 perf_event_print_debug();
1820                 intel_pmu_reset();
1821                 perf_enable();
1822                 return 1;
1823         }
1824
1825         inc_irq_stat(apic_perf_irqs);
1826         ack = status;
1827         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
1828                 struct perf_event *event = cpuc->events[bit];
1829
1830                 clear_bit(bit, (unsigned long *) &status);
1831                 if (!test_bit(bit, cpuc->active_mask))
1832                         continue;
1833
1834                 if (!intel_pmu_save_and_restart(event))
1835                         continue;
1836
1837                 data.period = event->hw.last_period;
1838
1839                 if (perf_event_overflow(event, 1, &data, regs))
1840                         intel_pmu_disable_event(&event->hw, bit);
1841         }
1842
1843         intel_pmu_ack_status(ack);
1844
1845         /*
1846          * Repeat if there is more work to be done:
1847          */
1848         status = intel_pmu_get_status();
1849         if (status)
1850                 goto again;
1851
1852         perf_enable();
1853
1854         return 1;
1855 }
1856
1857 static int amd_pmu_handle_irq(struct pt_regs *regs)
1858 {
1859         struct perf_sample_data data;
1860         struct cpu_hw_events *cpuc;
1861         struct perf_event *event;
1862         struct hw_perf_event *hwc;
1863         int idx, handled = 0;
1864         u64 val;
1865
1866         data.addr = 0;
1867         data.raw = NULL;
1868
1869         cpuc = &__get_cpu_var(cpu_hw_events);
1870
1871         for (idx = 0; idx < x86_pmu.num_events; idx++) {
1872                 if (!test_bit(idx, cpuc->active_mask))
1873                         continue;
1874
1875                 event = cpuc->events[idx];
1876                 hwc = &event->hw;
1877
1878                 val = x86_perf_event_update(event, hwc, idx);
1879                 if (val & (1ULL << (x86_pmu.event_bits - 1)))
1880                         continue;
1881
1882                 /*
1883                  * event overflow
1884                  */
1885                 handled         = 1;
1886                 data.period     = event->hw.last_period;
1887
1888                 if (!x86_perf_event_set_period(event, hwc, idx))
1889                         continue;
1890
1891                 if (perf_event_overflow(event, 1, &data, regs))
1892                         amd_pmu_disable_event(hwc, idx);
1893         }
1894
1895         if (handled)
1896                 inc_irq_stat(apic_perf_irqs);
1897
1898         return handled;
1899 }
1900
1901 void smp_perf_pending_interrupt(struct pt_regs *regs)
1902 {
1903         irq_enter();
1904         ack_APIC_irq();
1905         inc_irq_stat(apic_pending_irqs);
1906         perf_event_do_pending();
1907         irq_exit();
1908 }
1909
1910 void set_perf_event_pending(void)
1911 {
1912 #ifdef CONFIG_X86_LOCAL_APIC
1913         if (!x86_pmu.apic || !x86_pmu_initialized())
1914                 return;
1915
1916         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
1917 #endif
1918 }
1919
1920 void perf_events_lapic_init(void)
1921 {
1922 #ifdef CONFIG_X86_LOCAL_APIC
1923         if (!x86_pmu.apic || !x86_pmu_initialized())
1924                 return;
1925
1926         /*
1927          * Always use NMI for PMU
1928          */
1929         apic_write(APIC_LVTPC, APIC_DM_NMI);
1930 #endif
1931 }
1932
1933 static int __kprobes
1934 perf_event_nmi_handler(struct notifier_block *self,
1935                          unsigned long cmd, void *__args)
1936 {
1937         struct die_args *args = __args;
1938         struct pt_regs *regs;
1939
1940         if (!atomic_read(&active_events))
1941                 return NOTIFY_DONE;
1942
1943         switch (cmd) {
1944         case DIE_NMI:
1945         case DIE_NMI_IPI:
1946                 break;
1947
1948         default:
1949                 return NOTIFY_DONE;
1950         }
1951
1952         regs = args->regs;
1953
1954 #ifdef CONFIG_X86_LOCAL_APIC
1955         apic_write(APIC_LVTPC, APIC_DM_NMI);
1956 #endif
1957         /*
1958          * Can't rely on the handled return value to say it was our NMI, two
1959          * events could trigger 'simultaneously' raising two back-to-back NMIs.
1960          *
1961          * If the first NMI handles both, the latter will be empty and daze
1962          * the CPU.
1963          */
1964         x86_pmu.handle_irq(regs);
1965
1966         return NOTIFY_STOP;
1967 }
1968
1969 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
1970         .notifier_call          = perf_event_nmi_handler,
1971         .next                   = NULL,
1972         .priority               = 1
1973 };
1974
1975 static __initconst struct x86_pmu p6_pmu = {
1976         .name                   = "p6",
1977         .handle_irq             = p6_pmu_handle_irq,
1978         .disable_all            = p6_pmu_disable_all,
1979         .enable_all             = p6_pmu_enable_all,
1980         .enable                 = p6_pmu_enable_event,
1981         .disable                = p6_pmu_disable_event,
1982         .eventsel               = MSR_P6_EVNTSEL0,
1983         .perfctr                = MSR_P6_PERFCTR0,
1984         .event_map              = p6_pmu_event_map,
1985         .raw_event              = p6_pmu_raw_event,
1986         .max_events             = ARRAY_SIZE(p6_perfmon_event_map),
1987         .apic                   = 1,
1988         .max_period             = (1ULL << 31) - 1,
1989         .version                = 0,
1990         .num_events             = 2,
1991         /*
1992          * Events have 40 bits implemented. However they are designed such
1993          * that bits [32-39] are sign extensions of bit 31. As such the
1994          * effective width of a event for P6-like PMU is 32 bits only.
1995          *
1996          * See IA-32 Intel Architecture Software developer manual Vol 3B
1997          */
1998         .event_bits             = 32,
1999         .event_mask             = (1ULL << 32) - 1,
2000         .get_event_idx          = intel_get_event_idx,
2001 };
2002
2003 static __initconst struct x86_pmu intel_pmu = {
2004         .name                   = "Intel",
2005         .handle_irq             = intel_pmu_handle_irq,
2006         .disable_all            = intel_pmu_disable_all,
2007         .enable_all             = intel_pmu_enable_all,
2008         .enable                 = intel_pmu_enable_event,
2009         .disable                = intel_pmu_disable_event,
2010         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
2011         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
2012         .event_map              = intel_pmu_event_map,
2013         .raw_event              = intel_pmu_raw_event,
2014         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
2015         .apic                   = 1,
2016         /*
2017          * Intel PMCs cannot be accessed sanely above 32 bit width,
2018          * so we install an artificial 1<<31 period regardless of
2019          * the generic event period:
2020          */
2021         .max_period             = (1ULL << 31) - 1,
2022         .enable_bts             = intel_pmu_enable_bts,
2023         .disable_bts            = intel_pmu_disable_bts,
2024         .get_event_idx          = intel_get_event_idx,
2025 };
2026
2027 static __initconst struct x86_pmu amd_pmu = {
2028         .name                   = "AMD",
2029         .handle_irq             = amd_pmu_handle_irq,
2030         .disable_all            = amd_pmu_disable_all,
2031         .enable_all             = amd_pmu_enable_all,
2032         .enable                 = amd_pmu_enable_event,
2033         .disable                = amd_pmu_disable_event,
2034         .eventsel               = MSR_K7_EVNTSEL0,
2035         .perfctr                = MSR_K7_PERFCTR0,
2036         .event_map              = amd_pmu_event_map,
2037         .raw_event              = amd_pmu_raw_event,
2038         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
2039         .num_events             = 4,
2040         .event_bits             = 48,
2041         .event_mask             = (1ULL << 48) - 1,
2042         .apic                   = 1,
2043         /* use highest bit to detect overflow */
2044         .max_period             = (1ULL << 47) - 1,
2045         .get_event_idx          = gen_get_event_idx,
2046 };
2047
2048 static __init int p6_pmu_init(void)
2049 {
2050         switch (boot_cpu_data.x86_model) {
2051         case 1:
2052         case 3:  /* Pentium Pro */
2053         case 5:
2054         case 6:  /* Pentium II */
2055         case 7:
2056         case 8:
2057         case 11: /* Pentium III */
2058                 event_constraints = intel_p6_event_constraints;
2059                 break;
2060         case 9:
2061         case 13:
2062                 /* Pentium M */
2063                 event_constraints = intel_p6_event_constraints;
2064                 break;
2065         default:
2066                 pr_cont("unsupported p6 CPU model %d ",
2067                         boot_cpu_data.x86_model);
2068                 return -ENODEV;
2069         }
2070
2071         x86_pmu = p6_pmu;
2072
2073         return 0;
2074 }
2075
2076 static __init int intel_pmu_init(void)
2077 {
2078         union cpuid10_edx edx;
2079         union cpuid10_eax eax;
2080         unsigned int unused;
2081         unsigned int ebx;
2082         int version;
2083
2084         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) {
2085                 /* check for P6 processor family */
2086            if (boot_cpu_data.x86 == 6) {
2087                 return p6_pmu_init();
2088            } else {
2089                 return -ENODEV;
2090            }
2091         }
2092
2093         /*
2094          * Check whether the Architectural PerfMon supports
2095          * Branch Misses Retired hw_event or not.
2096          */
2097         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
2098         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
2099                 return -ENODEV;
2100
2101         version = eax.split.version_id;
2102         if (version < 2)
2103                 return -ENODEV;
2104
2105         x86_pmu                         = intel_pmu;
2106         x86_pmu.version                 = version;
2107         x86_pmu.num_events              = eax.split.num_events;
2108         x86_pmu.event_bits              = eax.split.bit_width;
2109         x86_pmu.event_mask              = (1ULL << eax.split.bit_width) - 1;
2110
2111         /*
2112          * Quirk: v2 perfmon does not report fixed-purpose events, so
2113          * assume at least 3 events:
2114          */
2115         x86_pmu.num_events_fixed        = max((int)edx.split.num_events_fixed, 3);
2116
2117         /*
2118          * Install the hw-cache-events table:
2119          */
2120         switch (boot_cpu_data.x86_model) {
2121         case 15: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
2122         case 22: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
2123         case 23: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
2124         case 29: /* six-core 45 nm xeon "Dunnington" */
2125                 memcpy(hw_cache_event_ids, core2_hw_cache_event_ids,
2126                        sizeof(hw_cache_event_ids));
2127
2128                 pr_cont("Core2 events, ");
2129                 event_constraints = intel_core_event_constraints;
2130                 break;
2131         default:
2132         case 26:
2133                 memcpy(hw_cache_event_ids, nehalem_hw_cache_event_ids,
2134                        sizeof(hw_cache_event_ids));
2135
2136                 event_constraints = intel_nehalem_event_constraints;
2137                 pr_cont("Nehalem/Corei7 events, ");
2138                 break;
2139         case 28:
2140                 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
2141                        sizeof(hw_cache_event_ids));
2142
2143                 pr_cont("Atom events, ");
2144                 break;
2145         }
2146         return 0;
2147 }
2148
2149 static __init int amd_pmu_init(void)
2150 {
2151         /* Performance-monitoring supported from K7 and later: */
2152         if (boot_cpu_data.x86 < 6)
2153                 return -ENODEV;
2154
2155         x86_pmu = amd_pmu;
2156
2157         /* Events are common for all AMDs */
2158         memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
2159                sizeof(hw_cache_event_ids));
2160
2161         return 0;
2162 }
2163
2164 static void __init pmu_check_apic(void)
2165 {
2166         if (cpu_has_apic)
2167                 return;
2168
2169         x86_pmu.apic = 0;
2170         pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
2171         pr_info("no hardware sampling interrupt available.\n");
2172 }
2173
2174 void __init init_hw_perf_events(void)
2175 {
2176         int err;
2177
2178         pr_info("Performance Events: ");
2179
2180         switch (boot_cpu_data.x86_vendor) {
2181         case X86_VENDOR_INTEL:
2182                 err = intel_pmu_init();
2183                 break;
2184         case X86_VENDOR_AMD:
2185                 err = amd_pmu_init();
2186                 break;
2187         default:
2188                 return;
2189         }
2190         if (err != 0) {
2191                 pr_cont("no PMU driver, software events only.\n");
2192                 return;
2193         }
2194
2195         pmu_check_apic();
2196
2197         pr_cont("%s PMU driver.\n", x86_pmu.name);
2198
2199         if (x86_pmu.num_events > X86_PMC_MAX_GENERIC) {
2200                 WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
2201                      x86_pmu.num_events, X86_PMC_MAX_GENERIC);
2202                 x86_pmu.num_events = X86_PMC_MAX_GENERIC;
2203         }
2204         perf_event_mask = (1 << x86_pmu.num_events) - 1;
2205         perf_max_events = x86_pmu.num_events;
2206
2207         if (x86_pmu.num_events_fixed > X86_PMC_MAX_FIXED) {
2208                 WARN(1, KERN_ERR "hw perf events fixed %d > max(%d), clipping!",
2209                      x86_pmu.num_events_fixed, X86_PMC_MAX_FIXED);
2210                 x86_pmu.num_events_fixed = X86_PMC_MAX_FIXED;
2211         }
2212
2213         perf_event_mask |=
2214                 ((1LL << x86_pmu.num_events_fixed)-1) << X86_PMC_IDX_FIXED;
2215         x86_pmu.intel_ctrl = perf_event_mask;
2216
2217         perf_events_lapic_init();
2218         register_die_notifier(&perf_event_nmi_notifier);
2219
2220         pr_info("... version:                %d\n",     x86_pmu.version);
2221         pr_info("... bit width:              %d\n",     x86_pmu.event_bits);
2222         pr_info("... generic registers:      %d\n",     x86_pmu.num_events);
2223         pr_info("... value mask:             %016Lx\n", x86_pmu.event_mask);
2224         pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
2225         pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_events_fixed);
2226         pr_info("... event mask:             %016Lx\n", perf_event_mask);
2227 }
2228
2229 static inline void x86_pmu_read(struct perf_event *event)
2230 {
2231         x86_perf_event_update(event, &event->hw, event->hw.idx);
2232 }
2233
2234 static const struct pmu pmu = {
2235         .enable         = x86_pmu_enable,
2236         .disable        = x86_pmu_disable,
2237         .read           = x86_pmu_read,
2238         .unthrottle     = x86_pmu_unthrottle,
2239 };
2240
2241 static int
2242 validate_event(struct cpu_hw_events *cpuc, struct perf_event *event)
2243 {
2244         struct hw_perf_event fake_event = event->hw;
2245
2246         if (event->pmu && event->pmu != &pmu)
2247                 return 0;
2248
2249         return x86_schedule_event(cpuc, &fake_event) >= 0;
2250 }
2251
2252 static int validate_group(struct perf_event *event)
2253 {
2254         struct perf_event *sibling, *leader = event->group_leader;
2255         struct cpu_hw_events fake_pmu;
2256
2257         memset(&fake_pmu, 0, sizeof(fake_pmu));
2258
2259         if (!validate_event(&fake_pmu, leader))
2260                 return -ENOSPC;
2261
2262         list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
2263                 if (!validate_event(&fake_pmu, sibling))
2264                         return -ENOSPC;
2265         }
2266
2267         if (!validate_event(&fake_pmu, event))
2268                 return -ENOSPC;
2269
2270         return 0;
2271 }
2272
2273 const struct pmu *hw_perf_event_init(struct perf_event *event)
2274 {
2275         int err;
2276
2277         err = __hw_perf_event_init(event);
2278         if (!err) {
2279                 if (event->group_leader != event)
2280                         err = validate_group(event);
2281         }
2282         if (err) {
2283                 if (event->destroy)
2284                         event->destroy(event);
2285                 return ERR_PTR(err);
2286         }
2287
2288         return &pmu;
2289 }
2290
2291 /*
2292  * callchain support
2293  */
2294
2295 static inline
2296 void callchain_store(struct perf_callchain_entry *entry, u64 ip)
2297 {
2298         if (entry->nr < PERF_MAX_STACK_DEPTH)
2299                 entry->ip[entry->nr++] = ip;
2300 }
2301
2302 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
2303 static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
2304 static DEFINE_PER_CPU(int, in_ignored_frame);
2305
2306
2307 static void
2308 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
2309 {
2310         /* Ignore warnings */
2311 }
2312
2313 static void backtrace_warning(void *data, char *msg)
2314 {
2315         /* Ignore warnings */
2316 }
2317
2318 static int backtrace_stack(void *data, char *name)
2319 {
2320         per_cpu(in_ignored_frame, smp_processor_id()) =
2321                         x86_is_stack_id(NMI_STACK, name) ||
2322                         x86_is_stack_id(DEBUG_STACK, name);
2323
2324         return 0;
2325 }
2326
2327 static void backtrace_address(void *data, unsigned long addr, int reliable)
2328 {
2329         struct perf_callchain_entry *entry = data;
2330
2331         if (per_cpu(in_ignored_frame, smp_processor_id()))
2332                 return;
2333
2334         if (reliable)
2335                 callchain_store(entry, addr);
2336 }
2337
2338 static const struct stacktrace_ops backtrace_ops = {
2339         .warning                = backtrace_warning,
2340         .warning_symbol         = backtrace_warning_symbol,
2341         .stack                  = backtrace_stack,
2342         .address                = backtrace_address,
2343         .walk_stack             = print_context_stack_bp,
2344 };
2345
2346 #include "../dumpstack.h"
2347
2348 static void
2349 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
2350 {
2351         callchain_store(entry, PERF_CONTEXT_KERNEL);
2352         callchain_store(entry, regs->ip);
2353
2354         dump_trace(NULL, regs, NULL, regs->bp, &backtrace_ops, entry);
2355 }
2356
2357 /*
2358  * best effort, GUP based copy_from_user() that assumes IRQ or NMI context
2359  */
2360 static unsigned long
2361 copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
2362 {
2363         unsigned long offset, addr = (unsigned long)from;
2364         int type = in_nmi() ? KM_NMI : KM_IRQ0;
2365         unsigned long size, len = 0;
2366         struct page *page;
2367         void *map;
2368         int ret;
2369
2370         do {
2371                 ret = __get_user_pages_fast(addr, 1, 0, &page);
2372                 if (!ret)
2373                         break;
2374
2375                 offset = addr & (PAGE_SIZE - 1);
2376                 size = min(PAGE_SIZE - offset, n - len);
2377
2378                 map = kmap_atomic(page, type);
2379                 memcpy(to, map+offset, size);
2380                 kunmap_atomic(map, type);
2381                 put_page(page);
2382
2383                 len  += size;
2384                 to   += size;
2385                 addr += size;
2386
2387         } while (len < n);
2388
2389         return len;
2390 }
2391
2392 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
2393 {
2394         unsigned long bytes;
2395
2396         bytes = copy_from_user_nmi(frame, fp, sizeof(*frame));
2397
2398         return bytes == sizeof(*frame);
2399 }
2400
2401 static void
2402 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
2403 {
2404         struct stack_frame frame;
2405         const void __user *fp;
2406
2407         if (!user_mode(regs))
2408                 regs = task_pt_regs(current);
2409
2410         fp = (void __user *)regs->bp;
2411
2412         callchain_store(entry, PERF_CONTEXT_USER);
2413         callchain_store(entry, regs->ip);
2414
2415         while (entry->nr < PERF_MAX_STACK_DEPTH) {
2416                 frame.next_frame             = NULL;
2417                 frame.return_address = 0;
2418
2419                 if (!copy_stack_frame(fp, &frame))
2420                         break;
2421
2422                 if ((unsigned long)fp < regs->sp)
2423                         break;
2424
2425                 callchain_store(entry, frame.return_address);
2426                 fp = frame.next_frame;
2427         }
2428 }
2429
2430 static void
2431 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
2432 {
2433         int is_user;
2434
2435         if (!regs)
2436                 return;
2437
2438         is_user = user_mode(regs);
2439
2440         if (!current || current->pid == 0)
2441                 return;
2442
2443         if (is_user && current->state != TASK_RUNNING)
2444                 return;
2445
2446         if (!is_user)
2447                 perf_callchain_kernel(regs, entry);
2448
2449         if (current->mm)
2450                 perf_callchain_user(regs, entry);
2451 }
2452
2453 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
2454 {
2455         struct perf_callchain_entry *entry;
2456
2457         if (in_nmi())
2458                 entry = &__get_cpu_var(pmc_nmi_entry);
2459         else
2460                 entry = &__get_cpu_var(pmc_irq_entry);
2461
2462         entry->nr = 0;
2463
2464         perf_do_callchain(regs, entry);
2465
2466         return entry;
2467 }
2468
2469 void hw_perf_event_setup_online(int cpu)
2470 {
2471         init_debug_store_on_cpu(cpu);
2472 }