Lock debugging enabled cause huge performance problems for tracing. Having
the lock verification happening for every function that is called
because mcount calls spin_lock can cripple the system.
This patch converts the spin_locks used by ftrace into raw_spin_locks.
Signed-off-by: Steven Rostedt <srostedt@redhat.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-static DEFINE_SPINLOCK(ftrace_max_lock);
+static raw_spinlock_t ftrace_max_lock =
+ (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
/*
* Copy the new maximum trace into the separate maximum-trace
/*
* Copy the new maximum trace into the separate maximum-trace
int i;
WARN_ON_ONCE(!irqs_disabled());
int i;
WARN_ON_ONCE(!irqs_disabled());
- spin_lock(&ftrace_max_lock);
+ __raw_spin_lock(&ftrace_max_lock);
/* clear out all the previous traces */
for_each_possible_cpu(i) {
data = tr->data[i];
/* clear out all the previous traces */
for_each_possible_cpu(i) {
data = tr->data[i];
}
__update_max_tr(tr, tsk, cpu);
}
__update_max_tr(tr, tsk, cpu);
- spin_unlock(&ftrace_max_lock);
+ __raw_spin_unlock(&ftrace_max_lock);
int i;
WARN_ON_ONCE(!irqs_disabled());
int i;
WARN_ON_ONCE(!irqs_disabled());
- spin_lock(&ftrace_max_lock);
+ __raw_spin_lock(&ftrace_max_lock);
for_each_possible_cpu(i)
tracing_reset(max_tr.data[i]);
for_each_possible_cpu(i)
tracing_reset(max_tr.data[i]);
tracing_reset(data);
__update_max_tr(tr, tsk, cpu);
tracing_reset(data);
__update_max_tr(tr, tsk, cpu);
- spin_unlock(&ftrace_max_lock);
+ __raw_spin_unlock(&ftrace_max_lock);
}
int register_tracer(struct tracer *type)
}
int register_tracer(struct tracer *type)
struct trace_entry *entry;
unsigned long irq_flags;
struct trace_entry *entry;
unsigned long irq_flags;
- spin_lock_irqsave(&data->lock, irq_flags);
+ raw_local_irq_save(irq_flags);
+ __raw_spin_lock(&data->lock);
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, flags);
entry->type = TRACE_FN;
entry->fn.ip = ip;
entry->fn.parent_ip = parent_ip;
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, flags);
entry->type = TRACE_FN;
entry->fn.ip = ip;
entry->fn.parent_ip = parent_ip;
- spin_unlock_irqrestore(&data->lock, irq_flags);
+ __raw_spin_unlock(&data->lock);
+ raw_local_irq_restore(irq_flags);
struct trace_entry *entry;
unsigned long irq_flags;
struct trace_entry *entry;
unsigned long irq_flags;
- spin_lock_irqsave(&data->lock, irq_flags);
+ raw_local_irq_save(irq_flags);
+ __raw_spin_lock(&data->lock);
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, 0);
entry->type = TRACE_SPECIAL;
entry->special.arg1 = arg1;
entry->special.arg2 = arg2;
entry->special.arg3 = arg3;
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, 0);
entry->type = TRACE_SPECIAL;
entry->special.arg1 = arg1;
entry->special.arg2 = arg2;
entry->special.arg3 = arg3;
- spin_unlock_irqrestore(&data->lock, irq_flags);
+ __raw_spin_unlock(&data->lock);
+ raw_local_irq_restore(irq_flags);
struct trace_entry *entry;
unsigned long irq_flags;
struct trace_entry *entry;
unsigned long irq_flags;
- spin_lock_irqsave(&data->lock, irq_flags);
+ raw_local_irq_save(irq_flags);
+ __raw_spin_lock(&data->lock);
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, flags);
entry->type = TRACE_CTX;
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, flags);
entry->type = TRACE_CTX;
entry->ctx.next_prio = next->prio;
entry->ctx.next_state = next->state;
__trace_stack(tr, data, flags, 4);
entry->ctx.next_prio = next->prio;
entry->ctx.next_state = next->state;
__trace_stack(tr, data, flags, 4);
- spin_unlock_irqrestore(&data->lock, irq_flags);
+ __raw_spin_unlock(&data->lock);
+ raw_local_irq_restore(irq_flags);
struct trace_entry *entry;
unsigned long irq_flags;
struct trace_entry *entry;
unsigned long irq_flags;
- spin_lock_irqsave(&data->lock, irq_flags);
+ raw_local_irq_save(irq_flags);
+ __raw_spin_lock(&data->lock);
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, flags);
entry->type = TRACE_WAKE;
entry = tracing_get_trace_entry(tr, data);
tracing_generic_entry_update(entry, flags);
entry->type = TRACE_WAKE;
entry->ctx.next_prio = wakee->prio;
entry->ctx.next_state = wakee->state;
__trace_stack(tr, data, flags, 5);
entry->ctx.next_prio = wakee->prio;
entry->ctx.next_state = wakee->state;
__trace_stack(tr, data, flags, 5);
- spin_unlock_irqrestore(&data->lock, irq_flags);
+ __raw_spin_unlock(&data->lock);
+ raw_local_irq_restore(irq_flags);
if (err)
goto err_unlock;
if (err)
goto err_unlock;
- spin_lock_irq(&ftrace_max_lock);
+ raw_local_irq_disable();
+ __raw_spin_lock(&ftrace_max_lock);
for_each_possible_cpu(cpu) {
/*
* Increase/decrease the disabled counter if we are
for_each_possible_cpu(cpu) {
/*
* Increase/decrease the disabled counter if we are
atomic_dec(&global_trace.data[cpu]->disabled);
}
}
atomic_dec(&global_trace.data[cpu]->disabled);
}
}
- spin_unlock_irq(&ftrace_max_lock);
+ __raw_spin_unlock(&ftrace_max_lock);
+ raw_local_irq_enable();
tracing_cpumask = tracing_cpumask_new;
tracing_cpumask = tracing_cpumask_new;
for_each_cpu_mask(cpu, mask) {
data = iter->tr->data[cpu];
for_each_cpu_mask(cpu, mask) {
data = iter->tr->data[cpu];
- spin_lock(&data->lock);
+ __raw_spin_lock(&data->lock);
}
while (find_next_entry_inc(iter) != NULL) {
}
while (find_next_entry_inc(iter) != NULL) {
for_each_cpu_mask(cpu, mask) {
data = iter->tr->data[cpu];
for_each_cpu_mask(cpu, mask) {
data = iter->tr->data[cpu];
- spin_unlock(&data->lock);
+ __raw_spin_unlock(&data->lock);
}
for_each_cpu_mask(cpu, mask) {
}
for_each_cpu_mask(cpu, mask) {
/* Now that we successfully allocate a page per CPU, add them */
for_each_possible_cpu(i) {
data = global_trace.data[i];
/* Now that we successfully allocate a page per CPU, add them */
for_each_possible_cpu(i) {
data = global_trace.data[i];
- spin_lock_init(&data->lock);
- lockdep_set_class(&data->lock, &data->lock_key);
+ data->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
page = list_entry(pages.next, struct page, lru);
list_del_init(&page->lru);
list_add_tail(&page->lru, &data->trace_pages);
page = list_entry(pages.next, struct page, lru);
list_del_init(&page->lru);
list_add_tail(&page->lru, &data->trace_pages);
#ifdef CONFIG_TRACER_MAX_TRACE
data = max_tr.data[i];
#ifdef CONFIG_TRACER_MAX_TRACE
data = max_tr.data[i];
- spin_lock_init(&data->lock);
- lockdep_set_class(&data->lock, &data->lock_key);
+ data->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
page = list_entry(pages.next, struct page, lru);
list_del_init(&page->lru);
list_add_tail(&page->lru, &data->trace_pages);
page = list_entry(pages.next, struct page, lru);
list_del_init(&page->lru);
list_add_tail(&page->lru, &data->trace_pages);
struct trace_array_cpu {
struct list_head trace_pages;
atomic_t disabled;
struct trace_array_cpu {
struct list_head trace_pages;
atomic_t disabled;
struct lock_class_key lock_key;
/* these fields get copied into max-trace: */
struct lock_class_key lock_key;
/* these fields get copied into max-trace: */