2 * Performance event support - Freescale Embedded Performance Monitor
4 * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
5 * Copyright 2010 Freescale Semiconductor, Inc.
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version.
12 #include <linux/kernel.h>
13 #include <linux/sched.h>
14 #include <linux/perf_event.h>
15 #include <linux/percpu.h>
16 #include <linux/hardirq.h>
17 #include <asm/reg_fsl_emb.h>
19 #include <asm/machdep.h>
20 #include <asm/firmware.h>
21 #include <asm/ptrace.h>
23 struct cpu_hw_events {
27 struct perf_event *event[MAX_HWEVENTS];
29 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
31 static struct fsl_emb_pmu *ppmu;
33 /* Number of perf_events counting hardware events */
34 static atomic_t num_events;
35 /* Used to avoid races in calling reserve/release_pmc_hardware */
36 static DEFINE_MUTEX(pmc_reserve_mutex);
39 * If interrupts were soft-disabled when a PMU interrupt occurs, treat
42 static inline int perf_intr_is_nmi(struct pt_regs *regs)
51 static void perf_event_interrupt(struct pt_regs *regs);
54 * Read one performance monitor counter (PMC).
56 static unsigned long read_pmc(int idx)
62 val = mfpmr(PMRN_PMC0);
65 val = mfpmr(PMRN_PMC1);
68 val = mfpmr(PMRN_PMC2);
71 val = mfpmr(PMRN_PMC3);
74 printk(KERN_ERR "oops trying to read PMC%d\n", idx);
83 static void write_pmc(int idx, unsigned long val)
87 mtpmr(PMRN_PMC0, val);
90 mtpmr(PMRN_PMC1, val);
93 mtpmr(PMRN_PMC2, val);
96 mtpmr(PMRN_PMC3, val);
99 printk(KERN_ERR "oops trying to write PMC%d\n", idx);
106 * Write one local control A register
108 static void write_pmlca(int idx, unsigned long val)
112 mtpmr(PMRN_PMLCA0, val);
115 mtpmr(PMRN_PMLCA1, val);
118 mtpmr(PMRN_PMLCA2, val);
121 mtpmr(PMRN_PMLCA3, val);
124 printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
131 * Write one local control B register
133 static void write_pmlcb(int idx, unsigned long val)
137 mtpmr(PMRN_PMLCB0, val);
140 mtpmr(PMRN_PMLCB1, val);
143 mtpmr(PMRN_PMLCB2, val);
146 mtpmr(PMRN_PMLCB3, val);
149 printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
155 static void fsl_emb_pmu_read(struct perf_event *event)
157 s64 val, delta, prev;
160 * Performance monitor interrupts come even when interrupts
161 * are soft-disabled, as long as interrupts are hard-enabled.
162 * Therefore we treat them like NMIs.
165 prev = atomic64_read(&event->hw.prev_count);
167 val = read_pmc(event->hw.idx);
168 } while (atomic64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
170 /* The counters are only 32 bits wide */
171 delta = (val - prev) & 0xfffffffful;
172 atomic64_add(delta, &event->count);
173 atomic64_sub(delta, &event->hw.period_left);
177 * Disable all events to prevent PMU interrupts and to allow
178 * events to be added or removed.
180 void hw_perf_disable(void)
182 struct cpu_hw_events *cpuhw;
185 local_irq_save(flags);
186 cpuhw = &__get_cpu_var(cpu_hw_events);
188 if (!cpuhw->disabled) {
192 * Check if we ever enabled the PMU on this cpu.
194 if (!cpuhw->pmcs_enabled) {
196 cpuhw->pmcs_enabled = 1;
199 if (atomic_read(&num_events)) {
201 * Set the 'freeze all counters' bit, and disable
202 * interrupts. The barrier is to make sure the
203 * mtpmr has been executed and the PMU has frozen
204 * the events before we return.
207 mtpmr(PMRN_PMGC0, PMGC0_FAC);
211 local_irq_restore(flags);
215 * Re-enable all events if disable == 0.
216 * If we were previously disabled and events were added, then
217 * put the new config on the PMU.
219 void hw_perf_enable(void)
221 struct cpu_hw_events *cpuhw;
224 local_irq_save(flags);
225 cpuhw = &__get_cpu_var(cpu_hw_events);
226 if (!cpuhw->disabled)
230 ppc_set_pmu_inuse(cpuhw->n_events != 0);
232 if (cpuhw->n_events > 0) {
233 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
238 local_irq_restore(flags);
241 static int collect_events(struct perf_event *group, int max_count,
242 struct perf_event *ctrs[])
245 struct perf_event *event;
247 if (!is_software_event(group)) {
253 list_for_each_entry(event, &group->sibling_list, group_entry) {
254 if (!is_software_event(event) &&
255 event->state != PERF_EVENT_STATE_OFF) {
265 /* perf must be disabled, context locked on entry */
266 static int fsl_emb_pmu_enable(struct perf_event *event)
268 struct cpu_hw_events *cpuhw;
270 int num_counters = ppmu->n_counter;
274 cpuhw = &get_cpu_var(cpu_hw_events);
276 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
277 num_counters = ppmu->n_restricted;
280 * Allocate counters from top-down, so that restricted-capable
281 * counters are kept free as long as possible.
283 for (i = num_counters - 1; i >= 0; i--) {
294 cpuhw->event[i] = event;
298 if (event->hw.sample_period) {
299 s64 left = atomic64_read(&event->hw.period_left);
300 if (left < 0x80000000L)
301 val = 0x80000000L - left;
303 atomic64_set(&event->hw.prev_count, val);
305 perf_event_update_userpage(event);
307 write_pmlcb(i, event->hw.config >> 32);
308 write_pmlca(i, event->hw.config_base);
312 put_cpu_var(cpu_hw_events);
316 /* perf must be disabled, context locked on entry */
317 static void fsl_emb_pmu_disable(struct perf_event *event)
319 struct cpu_hw_events *cpuhw;
320 int i = event->hw.idx;
325 fsl_emb_pmu_read(event);
327 cpuhw = &get_cpu_var(cpu_hw_events);
329 WARN_ON(event != cpuhw->event[event->hw.idx]);
335 cpuhw->event[i] = NULL;
339 * TODO: if at least one restricted event exists, and we
340 * just freed up a non-restricted-capable counter, and
341 * there is a restricted-capable counter occupied by
342 * a non-restricted event, migrate that event to the
349 put_cpu_var(cpu_hw_events);
353 * Re-enable interrupts on a event after they were throttled
354 * because they were coming too fast.
356 * Context is locked on entry, but perf is not disabled.
358 static void fsl_emb_pmu_unthrottle(struct perf_event *event)
363 if (event->hw.idx < 0 || !event->hw.sample_period)
365 local_irq_save(flags);
367 fsl_emb_pmu_read(event);
368 left = event->hw.sample_period;
369 event->hw.last_period = left;
371 if (left < 0x80000000L)
372 val = 0x80000000L - left;
373 write_pmc(event->hw.idx, val);
374 atomic64_set(&event->hw.prev_count, val);
375 atomic64_set(&event->hw.period_left, left);
376 perf_event_update_userpage(event);
378 local_irq_restore(flags);
381 static struct pmu fsl_emb_pmu = {
382 .enable = fsl_emb_pmu_enable,
383 .disable = fsl_emb_pmu_disable,
384 .read = fsl_emb_pmu_read,
385 .unthrottle = fsl_emb_pmu_unthrottle,
389 * Release the PMU if this is the last perf_event.
391 static void hw_perf_event_destroy(struct perf_event *event)
393 if (!atomic_add_unless(&num_events, -1, 1)) {
394 mutex_lock(&pmc_reserve_mutex);
395 if (atomic_dec_return(&num_events) == 0)
396 release_pmc_hardware();
397 mutex_unlock(&pmc_reserve_mutex);
402 * Translate a generic cache event_id config to a raw event_id code.
404 static int hw_perf_cache_event(u64 config, u64 *eventp)
406 unsigned long type, op, result;
409 if (!ppmu->cache_events)
413 type = config & 0xff;
414 op = (config >> 8) & 0xff;
415 result = (config >> 16) & 0xff;
417 if (type >= PERF_COUNT_HW_CACHE_MAX ||
418 op >= PERF_COUNT_HW_CACHE_OP_MAX ||
419 result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
422 ev = (*ppmu->cache_events)[type][op][result];
431 const struct pmu *hw_perf_event_init(struct perf_event *event)
434 struct perf_event *events[MAX_HWEVENTS];
440 switch (event->attr.type) {
441 case PERF_TYPE_HARDWARE:
442 ev = event->attr.config;
443 if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
444 return ERR_PTR(-EOPNOTSUPP);
445 ev = ppmu->generic_events[ev];
448 case PERF_TYPE_HW_CACHE:
449 err = hw_perf_cache_event(event->attr.config, &ev);
455 ev = event->attr.config;
459 return ERR_PTR(-EINVAL);
462 event->hw.config = ppmu->xlate_event(ev);
463 if (!(event->hw.config & FSL_EMB_EVENT_VALID))
464 return ERR_PTR(-EINVAL);
467 * If this is in a group, check if it can go on with all the
468 * other hardware events in the group. We assume the event
469 * hasn't been linked into its leader's sibling list at this point.
472 if (event->group_leader != event) {
473 n = collect_events(event->group_leader,
474 ppmu->n_counter - 1, events);
476 return ERR_PTR(-EINVAL);
479 if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
481 for (i = 0; i < n; i++) {
482 if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
486 if (num_restricted >= ppmu->n_restricted)
487 return ERR_PTR(-EINVAL);
492 event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
493 (u32)((ev << 16) & PMLCA_EVENT_MASK);
495 if (event->attr.exclude_user)
496 event->hw.config_base |= PMLCA_FCU;
497 if (event->attr.exclude_kernel)
498 event->hw.config_base |= PMLCA_FCS;
499 if (event->attr.exclude_idle)
500 return ERR_PTR(-ENOTSUPP);
502 event->hw.last_period = event->hw.sample_period;
503 atomic64_set(&event->hw.period_left, event->hw.last_period);
506 * See if we need to reserve the PMU.
507 * If no events are currently in use, then we have to take a
508 * mutex to ensure that we don't race with another task doing
509 * reserve_pmc_hardware or release_pmc_hardware.
512 if (!atomic_inc_not_zero(&num_events)) {
513 mutex_lock(&pmc_reserve_mutex);
514 if (atomic_read(&num_events) == 0 &&
515 reserve_pmc_hardware(perf_event_interrupt))
518 atomic_inc(&num_events);
519 mutex_unlock(&pmc_reserve_mutex);
521 mtpmr(PMRN_PMGC0, PMGC0_FAC);
524 event->destroy = hw_perf_event_destroy;
532 * A counter has overflowed; update its count and record
533 * things if requested. Note that interrupts are hard-disabled
534 * here so there is no possibility of being interrupted.
536 static void record_and_restart(struct perf_event *event, unsigned long val,
537 struct pt_regs *regs, int nmi)
539 u64 period = event->hw.sample_period;
540 s64 prev, delta, left;
543 /* we don't have to worry about interrupts here */
544 prev = atomic64_read(&event->hw.prev_count);
545 delta = (val - prev) & 0xfffffffful;
546 atomic64_add(delta, &event->count);
549 * See if the total period for this event has expired,
550 * and update for the next period.
553 left = atomic64_read(&event->hw.period_left) - delta;
561 if (left < 0x80000000LL)
562 val = 0x80000000LL - left;
566 * Finally record data if requested.
569 struct perf_sample_data data;
571 perf_sample_data_init(&data, 0);
573 if (perf_event_overflow(event, nmi, &data, regs)) {
575 * Interrupts are coming too fast - throttle them
576 * by setting the event to 0, so it will be
577 * at least 2^30 cycles until the next interrupt
578 * (assuming each event counts at most 2 counts
586 write_pmc(event->hw.idx, val);
587 atomic64_set(&event->hw.prev_count, val);
588 atomic64_set(&event->hw.period_left, left);
589 perf_event_update_userpage(event);
592 static void perf_event_interrupt(struct pt_regs *regs)
595 struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
596 struct perf_event *event;
601 nmi = perf_intr_is_nmi(regs);
607 for (i = 0; i < ppmu->n_counter; ++i) {
608 event = cpuhw->event[i];
613 /* event has overflowed */
615 record_and_restart(event, val, regs, nmi);
618 * Disabled counter is negative,
619 * reset it just in case.
626 /* PMM will keep counters frozen until we return from the interrupt. */
627 mtmsr(mfmsr() | MSR_PMM);
628 mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
637 void hw_perf_event_setup(int cpu)
639 struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
641 memset(cpuhw, 0, sizeof(*cpuhw));
644 int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
647 return -EBUSY; /* something's already registered */
650 pr_info("%s performance monitor hardware support registered\n",