4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 #include <linux/kallsyms.h>
7 #include <linux/seq_file.h>
8 #include <linux/spinlock.h>
9 #include <linux/irqflags.h>
10 #include <linux/uaccess.h>
11 #include <linux/module.h>
12 #include <linux/ftrace.h>
13 #include <linux/hash.h>
15 #include <asm/local.h>
18 #include "trace_stat.h"
19 #include "trace_output.h"
21 #ifdef CONFIG_BRANCH_TRACER
23 static struct tracer branch_trace;
24 static int branch_tracing_enabled __read_mostly;
25 static DEFINE_MUTEX(branch_tracing_mutex);
27 static struct trace_array *branch_tracer;
30 probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
32 struct trace_event_call *call = &event_branch;
33 struct trace_array *tr = branch_tracer;
34 struct trace_array_cpu *data;
35 struct ring_buffer_event *event;
36 struct trace_branch *entry;
37 struct ring_buffer *buffer;
42 if (current->trace_recursion & TRACE_BRANCH_BIT)
46 * I would love to save just the ftrace_likely_data pointer, but
47 * this code can also be used by modules. Ugly things can happen
48 * if the module is unloaded, and then we go and read the
49 * pointer. This is slower, but much safer.
55 raw_local_irq_save(flags);
56 current->trace_recursion |= TRACE_BRANCH_BIT;
57 data = this_cpu_ptr(tr->trace_buffer.data);
58 if (atomic_read(&data->disabled))
62 buffer = tr->trace_buffer.buffer;
63 event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH,
64 sizeof(*entry), flags, pc);
68 entry = ring_buffer_event_data(event);
70 /* Strip off the path, only save the file */
71 p = f->file + strlen(f->file);
72 while (p >= f->file && *p != '/')
76 strncpy(entry->func, f->func, TRACE_FUNC_SIZE);
77 strncpy(entry->file, p, TRACE_FILE_SIZE);
78 entry->func[TRACE_FUNC_SIZE] = 0;
79 entry->file[TRACE_FILE_SIZE] = 0;
80 entry->line = f->line;
81 entry->correct = val == expect;
83 if (!call_filter_check_discard(call, entry, buffer, event))
84 trace_buffer_unlock_commit_nostack(buffer, event);
87 current->trace_recursion &= ~TRACE_BRANCH_BIT;
88 raw_local_irq_restore(flags);
92 void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
94 if (!branch_tracing_enabled)
97 probe_likely_condition(f, val, expect);
100 int enable_branch_tracing(struct trace_array *tr)
102 mutex_lock(&branch_tracing_mutex);
105 * Must be seen before enabling. The reader is a condition
106 * where we do not need a matching rmb()
109 branch_tracing_enabled++;
110 mutex_unlock(&branch_tracing_mutex);
115 void disable_branch_tracing(void)
117 mutex_lock(&branch_tracing_mutex);
119 if (!branch_tracing_enabled)
122 branch_tracing_enabled--;
125 mutex_unlock(&branch_tracing_mutex);
128 static int branch_trace_init(struct trace_array *tr)
130 return enable_branch_tracing(tr);
133 static void branch_trace_reset(struct trace_array *tr)
135 disable_branch_tracing();
138 static enum print_line_t trace_branch_print(struct trace_iterator *iter,
139 int flags, struct trace_event *event)
141 struct trace_branch *field;
143 trace_assign_type(field, iter->ent);
145 trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
146 field->correct ? " ok " : " MISS ",
151 return trace_handle_return(&iter->seq);
154 static void branch_print_header(struct seq_file *s)
156 seq_puts(s, "# TASK-PID CPU# TIMESTAMP CORRECT"
162 static struct trace_event_functions trace_branch_funcs = {
163 .trace = trace_branch_print,
166 static struct trace_event trace_branch_event = {
167 .type = TRACE_BRANCH,
168 .funcs = &trace_branch_funcs,
171 static struct tracer branch_trace __read_mostly =
174 .init = branch_trace_init,
175 .reset = branch_trace_reset,
176 #ifdef CONFIG_FTRACE_SELFTEST
177 .selftest = trace_selftest_startup_branch,
178 #endif /* CONFIG_FTRACE_SELFTEST */
179 .print_header = branch_print_header,
182 __init static int init_branch_tracer(void)
186 ret = register_trace_event(&trace_branch_event);
188 printk(KERN_WARNING "Warning: could not register "
192 return register_tracer(&branch_trace);
194 core_initcall(init_branch_tracer);
198 void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
201 #endif /* CONFIG_BRANCH_TRACER */
203 void ftrace_likely_update(struct ftrace_likely_data *f, int val,
204 int expect, int is_constant)
206 /* A constant is always correct */
212 * I would love to have a trace point here instead, but the
213 * trace point code is so inundated with unlikely and likely
214 * conditions that the recursive nightmare that exists is too
215 * much to try to get working. At least for now.
217 trace_likely_condition(&f->data, val, expect);
219 /* FIXME: Make this atomic! */
225 EXPORT_SYMBOL(ftrace_likely_update);
227 extern unsigned long __start_annotated_branch_profile[];
228 extern unsigned long __stop_annotated_branch_profile[];
230 static int annotated_branch_stat_headers(struct seq_file *m)
232 seq_puts(m, " correct incorrect % "
235 " ------- --------- - "
241 static inline long get_incorrect_percent(struct ftrace_branch_data *p)
246 percent = p->incorrect * 100;
247 percent /= p->correct + p->incorrect;
249 percent = p->incorrect ? 100 : -1;
254 static const char *branch_stat_process_file(struct ftrace_branch_data *p)
258 /* Only print the file, not the path */
259 f = p->file + strlen(p->file);
260 while (f >= p->file && *f != '/')
265 static void branch_stat_show(struct seq_file *m,
266 struct ftrace_branch_data *p, const char *f)
271 * The miss is overlayed on correct, and hit on incorrect.
273 percent = get_incorrect_percent(p);
278 seq_printf(m, "%3ld ", percent);
280 seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
283 static int branch_stat_show_normal(struct seq_file *m,
284 struct ftrace_branch_data *p, const char *f)
286 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
287 branch_stat_show(m, p, f);
291 static int annotate_branch_stat_show(struct seq_file *m, void *v)
293 struct ftrace_likely_data *p = v;
297 f = branch_stat_process_file(&p->data);
300 return branch_stat_show_normal(m, &p->data, f);
302 l = snprintf(NULL, 0, "/%lu", p->constant);
303 l = l > 8 ? 0 : 8 - l;
305 seq_printf(m, "%8lu/%lu %*lu ",
306 p->data.correct, p->constant, l, p->data.incorrect);
307 branch_stat_show(m, &p->data, f);
311 static void *annotated_branch_stat_start(struct tracer_stat *trace)
313 return __start_annotated_branch_profile;
317 annotated_branch_stat_next(void *v, int idx)
319 struct ftrace_likely_data *p = v;
323 if ((void *)p >= (void *)__stop_annotated_branch_profile)
329 static int annotated_branch_stat_cmp(void *p1, void *p2)
331 struct ftrace_branch_data *a = p1;
332 struct ftrace_branch_data *b = p2;
334 long percent_a, percent_b;
336 percent_a = get_incorrect_percent(a);
337 percent_b = get_incorrect_percent(b);
339 if (percent_a < percent_b)
341 if (percent_a > percent_b)
344 if (a->incorrect < b->incorrect)
346 if (a->incorrect > b->incorrect)
350 * Since the above shows worse (incorrect) cases
351 * first, we continue that by showing best (correct)
354 if (a->correct > b->correct)
356 if (a->correct < b->correct)
362 static struct tracer_stat annotated_branch_stats = {
363 .name = "branch_annotated",
364 .stat_start = annotated_branch_stat_start,
365 .stat_next = annotated_branch_stat_next,
366 .stat_cmp = annotated_branch_stat_cmp,
367 .stat_headers = annotated_branch_stat_headers,
368 .stat_show = annotate_branch_stat_show
371 __init static int init_annotated_branch_stats(void)
375 ret = register_stat_tracer(&annotated_branch_stats);
377 printk(KERN_WARNING "Warning: could not register "
378 "annotated branches stats\n");
383 fs_initcall(init_annotated_branch_stats);
385 #ifdef CONFIG_PROFILE_ALL_BRANCHES
387 extern unsigned long __start_branch_profile[];
388 extern unsigned long __stop_branch_profile[];
390 static int all_branch_stat_headers(struct seq_file *m)
392 seq_puts(m, " miss hit % "
395 " ------- --------- - "
401 static void *all_branch_stat_start(struct tracer_stat *trace)
403 return __start_branch_profile;
407 all_branch_stat_next(void *v, int idx)
409 struct ftrace_branch_data *p = v;
413 if ((void *)p >= (void *)__stop_branch_profile)
419 static int all_branch_stat_show(struct seq_file *m, void *v)
421 struct ftrace_branch_data *p = v;
424 f = branch_stat_process_file(p);
425 return branch_stat_show_normal(m, p, f);
428 static struct tracer_stat all_branch_stats = {
429 .name = "branch_all",
430 .stat_start = all_branch_stat_start,
431 .stat_next = all_branch_stat_next,
432 .stat_headers = all_branch_stat_headers,
433 .stat_show = all_branch_stat_show
436 __init static int all_annotated_branch_stats(void)
440 ret = register_stat_tracer(&all_branch_stats);
442 printk(KERN_WARNING "Warning: could not register "
443 "all branches stats\n");
448 fs_initcall(all_annotated_branch_stats);
449 #endif /* CONFIG_PROFILE_ALL_BRANCHES */