For kernels built with PREEMPT_RT, critical sections protected
by standard spinlocks are preemptible. This is not acceptable
on perf as (a) we may be scheduled onto a different CPU whilst
reading/writing banked PMU registers and (b) the latency when
reading the PMU registers becomes unpredictable.
This patch upgrades the pmu_lock spinlock to a raw_spinlock
instead.
Reported-by: Jamie Iles <jamie@jamieiles.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
* Hardware lock to serialize accesses to PMU registers. Needed for the
* read/modify/write sequences.
*/
* Hardware lock to serialize accesses to PMU registers. Needed for the
* read/modify/write sequences.
*/
-static DEFINE_SPINLOCK(pmu_lock);
+static DEFINE_RAW_SPINLOCK(pmu_lock);
/*
* ARMv6 supports a maximum of 3 events, starting from index 1. If we add
/*
* ARMv6 supports a maximum of 3 events, starting from index 1. If we add
* Mask out the current event and set the counter to count the event
* that we're interested in.
*/
* Mask out the current event and set the counter to count the event
* that we're interested in.
*/
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = armv6_pmcr_read();
val &= ~mask;
val |= evt;
armv6_pmcr_write(val);
val = armv6_pmcr_read();
val &= ~mask;
val |= evt;
armv6_pmcr_write(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
{
unsigned long flags, val;
{
unsigned long flags, val;
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = armv6_pmcr_read();
val |= ARMV6_PMCR_ENABLE;
armv6_pmcr_write(val);
val = armv6_pmcr_read();
val |= ARMV6_PMCR_ENABLE;
armv6_pmcr_write(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
{
unsigned long flags, val;
{
unsigned long flags, val;
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = armv6_pmcr_read();
val &= ~ARMV6_PMCR_ENABLE;
armv6_pmcr_write(val);
val = armv6_pmcr_read();
val &= ~ARMV6_PMCR_ENABLE;
armv6_pmcr_write(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
* of ETM bus signal assertion cycles. The external reporting should
* be disabled and so this should never increment.
*/
* of ETM bus signal assertion cycles. The external reporting should
* be disabled and so this should never increment.
*/
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = armv6_pmcr_read();
val &= ~mask;
val |= evt;
armv6_pmcr_write(val);
val = armv6_pmcr_read();
val &= ~mask;
val |= evt;
armv6_pmcr_write(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
* Unlike UP ARMv6, we don't have a way of stopping the counters. We
* simply disable the interrupt reporting.
*/
* Unlike UP ARMv6, we don't have a way of stopping the counters. We
* simply disable the interrupt reporting.
*/
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = armv6_pmcr_read();
val &= ~mask;
val |= evt;
armv6_pmcr_write(val);
val = armv6_pmcr_read();
val &= ~mask;
val |= evt;
armv6_pmcr_write(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
}
static const struct arm_pmu armv6pmu = {
}
static const struct arm_pmu armv6pmu = {
* Enable counter and interrupt, and set the counter to count
* the event that we're interested in.
*/
* Enable counter and interrupt, and set the counter to count
* the event that we're interested in.
*/
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
*/
armv7_pmnc_enable_counter(idx);
*/
armv7_pmnc_enable_counter(idx);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
}
static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
}
static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
/*
* Disable counter and interrupt
*/
/*
* Disable counter and interrupt
*/
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
*/
armv7_pmnc_disable_intens(idx);
*/
armv7_pmnc_disable_intens(idx);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
}
static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
}
static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
/* Enable all counters */
armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
/* Enable all counters */
armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
}
static void armv7pmu_stop(void)
{
unsigned long flags;
}
static void armv7pmu_stop(void)
{
unsigned long flags;
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
/* Disable all counters */
armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
/* Disable all counters */
armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
}
static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
}
static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = xscale1pmu_read_pmnc();
val &= ~mask;
val |= evt;
xscale1pmu_write_pmnc(val);
val = xscale1pmu_read_pmnc();
val &= ~mask;
val |= evt;
xscale1pmu_write_pmnc(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = xscale1pmu_read_pmnc();
val &= ~mask;
val |= evt;
xscale1pmu_write_pmnc(val);
val = xscale1pmu_read_pmnc();
val &= ~mask;
val |= evt;
xscale1pmu_write_pmnc(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
{
unsigned long flags, val;
{
unsigned long flags, val;
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = xscale1pmu_read_pmnc();
val |= XSCALE_PMU_ENABLE;
xscale1pmu_write_pmnc(val);
val = xscale1pmu_read_pmnc();
val |= XSCALE_PMU_ENABLE;
xscale1pmu_write_pmnc(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
{
unsigned long flags, val;
{
unsigned long flags, val;
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = xscale1pmu_read_pmnc();
val &= ~XSCALE_PMU_ENABLE;
xscale1pmu_write_pmnc(val);
val = xscale1pmu_read_pmnc();
val &= ~XSCALE_PMU_ENABLE;
xscale1pmu_write_pmnc(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
xscale2pmu_write_event_select(evtsel);
xscale2pmu_write_int_enable(ien);
xscale2pmu_write_event_select(evtsel);
xscale2pmu_write_int_enable(ien);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
xscale2pmu_write_event_select(evtsel);
xscale2pmu_write_int_enable(ien);
xscale2pmu_write_event_select(evtsel);
xscale2pmu_write_int_enable(ien);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
{
unsigned long flags, val;
{
unsigned long flags, val;
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
val |= XSCALE_PMU_ENABLE;
xscale2pmu_write_pmnc(val);
val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
val |= XSCALE_PMU_ENABLE;
xscale2pmu_write_pmnc(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);
{
unsigned long flags, val;
{
unsigned long flags, val;
- spin_lock_irqsave(&pmu_lock, flags);
+ raw_spin_lock_irqsave(&pmu_lock, flags);
val = xscale2pmu_read_pmnc();
val &= ~XSCALE_PMU_ENABLE;
xscale2pmu_write_pmnc(val);
val = xscale2pmu_read_pmnc();
val &= ~XSCALE_PMU_ENABLE;
xscale2pmu_write_pmnc(val);
- spin_unlock_irqrestore(&pmu_lock, flags);
+ raw_spin_unlock_irqrestore(&pmu_lock, flags);