2 * Linux performance counter support for MIPS.
4 * Copyright (C) 2010 MIPS Technologies, Inc.
5 * Author: Deng-Cheng Zhu
7 * This code is based on the implementation for ARM, which is in turn
8 * based on the sparc64 perf event code and the x86 code. Performance
9 * counter access is based on the MIPS Oprofile code.
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License version 2 as
13 * published by the Free Software Foundation.
16 #include <linux/cpumask.h>
17 #include <linux/interrupt.h>
18 #include <linux/smp.h>
19 #include <linux/kernel.h>
20 #include <linux/perf_event.h>
21 #include <linux/uaccess.h>
24 #include <asm/irq_regs.h>
25 #include <asm/stacktrace.h>
26 #include <asm/time.h> /* For perf_irq */
28 /* These are for 32bit counters. For 64bit ones, define them accordingly. */
29 #define MAX_PERIOD ((1ULL << 32) - 1)
30 #define VALID_COUNT 0x7fffffff
32 #define HIGHEST_BIT 31
34 #define MIPS_MAX_HWEVENTS 4
36 struct cpu_hw_events {
37 /* Array of events on this cpu. */
38 struct perf_event *events[MIPS_MAX_HWEVENTS];
41 * Set the bit (indexed by the counter number) when the counter
42 * is used for an event.
44 unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
47 * The borrowed MSB for the performance counter. A MIPS performance
48 * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit
49 * counters) as a factor of determining whether a counter overflow
50 * should be signaled. So here we use a separate MSB for each
51 * counter to make things easy.
53 unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)];
56 * Software copy of the control register for each performance counter.
57 * MIPS CPUs vary in performance counters. They use this differently,
58 * and even may not use it.
60 unsigned int saved_ctrl[MIPS_MAX_HWEVENTS];
62 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
66 /* The description of MIPS performance events. */
67 struct mips_perf_event {
68 unsigned int event_id;
70 * MIPS performance counters are indexed starting from 0.
71 * CNTR_EVEN indicates the indexes of the counters to be used are
74 unsigned int cntr_mask;
75 #define CNTR_EVEN 0x55555555
76 #define CNTR_ODD 0xaaaaaaaa
77 #ifdef CONFIG_MIPS_MT_SMP
90 #define UNSUPPORTED_PERF_EVENT_ID 0xffffffff
91 #define C(x) PERF_COUNT_HW_CACHE_##x
96 irqreturn_t (*handle_irq)(int irq, void *dev);
97 int (*handle_shared_irq)(void);
100 int (*alloc_counter)(struct cpu_hw_events *cpuc,
101 struct hw_perf_event *hwc);
102 u64 (*read_counter)(unsigned int idx);
103 void (*write_counter)(unsigned int idx, u64 val);
104 void (*enable_event)(struct hw_perf_event *evt, int idx);
105 void (*disable_event)(int idx);
106 const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX];
107 const struct mips_perf_event (*cache_event_map)
108 [PERF_COUNT_HW_CACHE_MAX]
109 [PERF_COUNT_HW_CACHE_OP_MAX]
110 [PERF_COUNT_HW_CACHE_RESULT_MAX];
111 unsigned int num_counters;
114 static const struct mips_pmu *mipspmu;
117 mipspmu_event_set_period(struct perf_event *event,
118 struct hw_perf_event *hwc,
121 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
122 s64 left = local64_read(&hwc->period_left);
123 s64 period = hwc->sample_period;
128 if (unlikely(left <= -period)) {
130 local64_set(&hwc->period_left, left);
131 hwc->last_period = period;
135 if (unlikely(left <= 0)) {
137 local64_set(&hwc->period_left, left);
138 hwc->last_period = period;
142 if (left > (s64)MAX_PERIOD)
145 local64_set(&hwc->prev_count, (u64)-left);
147 local_irq_save(flags);
148 uleft = (u64)(-left) & MAX_PERIOD;
149 uleft > VALID_COUNT ?
150 set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs);
151 mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT);
152 local_irq_restore(flags);
154 perf_event_update_userpage(event);
159 static int mipspmu_enable(struct perf_event *event)
161 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
162 struct hw_perf_event *hwc = &event->hw;
166 /* To look for a free counter for this event. */
167 idx = mipspmu->alloc_counter(cpuc, hwc);
174 * If there is an event in the counter we are going to use then
175 * make sure it is disabled.
178 mipspmu->disable_event(idx);
179 cpuc->events[idx] = event;
181 /* Set the period for the event. */
182 mipspmu_event_set_period(event, hwc, idx);
184 /* Enable the event. */
185 mipspmu->enable_event(hwc, idx);
187 /* Propagate our changes to the userspace mapping. */
188 perf_event_update_userpage(event);
194 static void mipspmu_event_update(struct perf_event *event,
195 struct hw_perf_event *hwc,
198 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
200 int shift = 64 - TOTAL_BITS;
201 s64 prev_raw_count, new_raw_count;
205 prev_raw_count = local64_read(&hwc->prev_count);
206 local_irq_save(flags);
207 /* Make the counter value be a "real" one. */
208 new_raw_count = mipspmu->read_counter(idx);
209 if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) {
210 new_raw_count &= VALID_COUNT;
211 clear_bit(idx, cpuc->msbs);
213 new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT);
214 local_irq_restore(flags);
216 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
217 new_raw_count) != prev_raw_count)
220 delta = (new_raw_count << shift) - (prev_raw_count << shift);
223 local64_add(delta, &event->count);
224 local64_sub(delta, &hwc->period_left);
229 static void mipspmu_disable(struct perf_event *event)
231 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
232 struct hw_perf_event *hwc = &event->hw;
236 WARN_ON(idx < 0 || idx >= mipspmu->num_counters);
238 /* We are working on a local event. */
239 mipspmu->disable_event(idx);
243 mipspmu_event_update(event, hwc, idx);
244 cpuc->events[idx] = NULL;
245 clear_bit(idx, cpuc->used_mask);
247 perf_event_update_userpage(event);
250 static void mipspmu_unthrottle(struct perf_event *event)
252 struct hw_perf_event *hwc = &event->hw;
254 mipspmu->enable_event(hwc, hwc->idx);
257 static void mipspmu_read(struct perf_event *event)
259 struct hw_perf_event *hwc = &event->hw;
261 /* Don't read disabled counters! */
265 mipspmu_event_update(event, hwc, hwc->idx);
268 static struct pmu pmu = {
269 .enable = mipspmu_enable,
270 .disable = mipspmu_disable,
271 .unthrottle = mipspmu_unthrottle,
272 .read = mipspmu_read,
275 static atomic_t active_events = ATOMIC_INIT(0);
276 static DEFINE_MUTEX(pmu_reserve_mutex);
277 static int (*save_perf_irq)(void);
279 static int mipspmu_get_irq(void)
283 if (mipspmu->irq >= 0) {
284 /* Request my own irq handler. */
285 err = request_irq(mipspmu->irq, mipspmu->handle_irq,
286 IRQF_DISABLED | IRQF_NOBALANCING,
287 "mips_perf_pmu", NULL);
289 pr_warning("Unable to request IRQ%d for MIPS "
290 "performance counters!\n", mipspmu->irq);
292 } else if (cp0_perfcount_irq < 0) {
294 * We are sharing the irq number with the timer interrupt.
296 save_perf_irq = perf_irq;
297 perf_irq = mipspmu->handle_shared_irq;
300 pr_warning("The platform hasn't properly defined its "
301 "interrupt controller.\n");
308 static void mipspmu_free_irq(void)
310 if (mipspmu->irq >= 0)
311 free_irq(mipspmu->irq, NULL);
312 else if (cp0_perfcount_irq < 0)
313 perf_irq = save_perf_irq;
316 static inline unsigned int
317 mipspmu_perf_event_encode(const struct mips_perf_event *pev)
320 * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for
323 #ifdef CONFIG_MIPS_MT_SMP
324 return ((unsigned int)pev->range << 24) |
325 (pev->cntr_mask & 0xffff00) |
326 (pev->event_id & 0xff);
328 return (pev->cntr_mask & 0xffff00) |
329 (pev->event_id & 0xff);
333 static const struct mips_perf_event *
334 mipspmu_map_general_event(int idx)
336 const struct mips_perf_event *pev;
338 pev = ((*mipspmu->general_event_map)[idx].event_id ==
339 UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) :
340 &(*mipspmu->general_event_map)[idx]);
345 static const struct mips_perf_event *
346 mipspmu_map_cache_event(u64 config)
348 unsigned int cache_type, cache_op, cache_result;
349 const struct mips_perf_event *pev;
351 cache_type = (config >> 0) & 0xff;
352 if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
353 return ERR_PTR(-EINVAL);
355 cache_op = (config >> 8) & 0xff;
356 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
357 return ERR_PTR(-EINVAL);
359 cache_result = (config >> 16) & 0xff;
360 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
361 return ERR_PTR(-EINVAL);
363 pev = &((*mipspmu->cache_event_map)
368 if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID)
369 return ERR_PTR(-EOPNOTSUPP);
375 static int validate_event(struct cpu_hw_events *cpuc,
376 struct perf_event *event)
378 struct hw_perf_event fake_hwc = event->hw;
380 if (event->pmu && event->pmu != &pmu)
383 return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0;
386 static int validate_group(struct perf_event *event)
388 struct perf_event *sibling, *leader = event->group_leader;
389 struct cpu_hw_events fake_cpuc;
391 memset(&fake_cpuc, 0, sizeof(fake_cpuc));
393 if (!validate_event(&fake_cpuc, leader))
396 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
397 if (!validate_event(&fake_cpuc, sibling))
401 if (!validate_event(&fake_cpuc, event))
408 * mipsxx/rm9000/loongson2 have different performance counters, they have
409 * specific low-level init routines.
411 static int __hw_perf_event_init(struct perf_event *event);
413 static void hw_perf_event_destroy(struct perf_event *event)
415 if (atomic_dec_and_mutex_lock(&active_events,
416 &pmu_reserve_mutex)) {
418 * We must not call the destroy function with interrupts
421 on_each_cpu(reset_counters,
422 (void *)(long)mipspmu->num_counters, 1);
424 mutex_unlock(&pmu_reserve_mutex);
428 const struct pmu *hw_perf_event_init(struct perf_event *event)
432 if (!mipspmu || event->cpu >= nr_cpumask_bits ||
433 (event->cpu >= 0 && !cpu_online(event->cpu)))
434 return ERR_PTR(-ENODEV);
436 if (!atomic_inc_not_zero(&active_events)) {
437 if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) {
438 atomic_dec(&active_events);
439 return ERR_PTR(-ENOSPC);
442 mutex_lock(&pmu_reserve_mutex);
443 if (atomic_read(&active_events) == 0)
444 err = mipspmu_get_irq();
447 atomic_inc(&active_events);
448 mutex_unlock(&pmu_reserve_mutex);
454 err = __hw_perf_event_init(event);
456 hw_perf_event_destroy(event);
458 return err ? ERR_PTR(err) : &pmu;
461 void hw_perf_enable(void)
467 void hw_perf_disable(void)
473 /* This is needed by specific irq handlers in perf_event_*.c */
475 handle_associated_event(struct cpu_hw_events *cpuc,
476 int idx, struct perf_sample_data *data, struct pt_regs *regs)
478 struct perf_event *event = cpuc->events[idx];
479 struct hw_perf_event *hwc = &event->hw;
481 mipspmu_event_update(event, hwc, idx);
482 data->period = event->hw.last_period;
483 if (!mipspmu_event_set_period(event, hwc, idx))
486 if (perf_event_overflow(event, 0, data, regs))
487 mipspmu->disable_event(idx);