4 * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 * Copyright (C) 2004, 2005, Soeren Sandmann
8 #include <linux/kallsyms.h>
9 #include <linux/debugfs.h>
10 #include <linux/hrtimer.h>
11 #include <linux/uaccess.h>
12 #include <linux/ftrace.h>
13 #include <linux/module.h>
14 #include <linux/irq.h>
17 #include <asm/stacktrace.h>
21 static struct trace_array *sysprof_trace;
22 static int __read_mostly tracer_enabled;
25 * 1 msec sample interval by default:
27 static unsigned long sample_period = 1000000;
28 static const unsigned int sample_max_depth = 512;
30 static DEFINE_MUTEX(sample_timer_lock);
32 * Per CPU hrtimers that do the profiling:
34 static DEFINE_PER_CPU(struct hrtimer, stack_trace_hrtimer);
37 const void __user *next_fp;
38 unsigned long return_address;
41 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
45 if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
50 if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
57 struct backtrace_info {
58 struct trace_array_cpu *data;
59 struct trace_array *tr;
64 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
69 static void backtrace_warning(void *data, char *msg)
74 static int backtrace_stack(void *data, char *name)
76 /* Don't bother with IRQ stacks for now */
80 static void backtrace_address(void *data, unsigned long addr, int reliable)
82 struct backtrace_info *info = data;
84 if (info->pos < sample_max_depth && reliable) {
85 __trace_special(info->tr, info->data, 1, addr, 0);
91 const static struct stacktrace_ops backtrace_ops = {
92 .warning = backtrace_warning,
93 .warning_symbol = backtrace_warning_symbol,
94 .stack = backtrace_stack,
95 .address = backtrace_address,
98 static struct pt_regs *
99 trace_kernel(struct pt_regs *regs, struct trace_array *tr,
100 struct trace_array_cpu *data)
102 struct backtrace_info info;
111 __trace_special(info.tr, info.data, 1, regs->ip, 0);
113 stack = ((char *)regs + sizeof(struct pt_regs));
114 #ifdef CONFIG_FRAME_POINTER
120 dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, &info);
122 /* Now trace the user stack */
123 user_stack = ((char *)current->thread.sp0 - sizeof(struct pt_regs));
125 return (struct pt_regs *)user_stack;
128 static void timer_notify(struct pt_regs *regs, int cpu)
130 struct trace_array_cpu *data;
131 struct stack_frame frame;
132 struct trace_array *tr;
133 const void __user *fp;
141 data = tr->data[cpu];
142 is_user = user_mode(regs);
144 if (!current || current->pid == 0)
147 if (is_user && current->state != TASK_RUNNING)
150 __trace_special(tr, data, 0, 0, current->pid);
153 regs = trace_kernel(regs, tr, data);
155 fp = (void __user *)regs->bp;
157 __trace_special(tr, data, 2, regs->ip, 0);
159 for (i = 0; i < sample_max_depth; i++) {
161 frame.return_address = 0;
162 if (!copy_stack_frame(fp, &frame))
164 if ((unsigned long)fp < regs->sp)
167 __trace_special(tr, data, 2, frame.return_address,
172 __trace_special(tr, data, 3, current->pid, i);
175 * Special trace entry if we overflow the max depth:
177 if (i == sample_max_depth)
178 __trace_special(tr, data, -1, -1, -1);
181 static enum hrtimer_restart stack_trace_timer_fn(struct hrtimer *hrtimer)
184 timer_notify(get_irq_regs(), smp_processor_id());
186 hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
188 return HRTIMER_RESTART;
191 static void start_stack_timer(int cpu)
193 struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
195 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
196 hrtimer->function = stack_trace_timer_fn;
197 hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
199 hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL);
202 static void start_stack_timers(void)
204 cpumask_t saved_mask = current->cpus_allowed;
207 for_each_online_cpu(cpu) {
208 set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
209 start_stack_timer(cpu);
211 set_cpus_allowed_ptr(current, &saved_mask);
214 static void stop_stack_timer(int cpu)
216 struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
218 hrtimer_cancel(hrtimer);
221 static void stop_stack_timers(void)
225 for_each_online_cpu(cpu)
226 stop_stack_timer(cpu);
229 static void stack_reset(struct trace_array *tr)
233 tr->time_start = ftrace_now(tr->cpu);
235 for_each_online_cpu(cpu)
236 tracing_reset(tr->data[cpu]);
239 static void start_stack_trace(struct trace_array *tr)
241 mutex_lock(&sample_timer_lock);
243 start_stack_timers();
245 mutex_unlock(&sample_timer_lock);
248 static void stop_stack_trace(struct trace_array *tr)
250 mutex_lock(&sample_timer_lock);
253 mutex_unlock(&sample_timer_lock);
256 static void stack_trace_init(struct trace_array *tr)
261 start_stack_trace(tr);
264 static void stack_trace_reset(struct trace_array *tr)
267 stop_stack_trace(tr);
270 static void stack_trace_ctrl_update(struct trace_array *tr)
272 /* When starting a new trace, reset the buffers */
274 start_stack_trace(tr);
276 stop_stack_trace(tr);
279 static struct tracer stack_trace __read_mostly =
282 .init = stack_trace_init,
283 .reset = stack_trace_reset,
284 .ctrl_update = stack_trace_ctrl_update,
285 #ifdef CONFIG_FTRACE_SELFTEST
286 .selftest = trace_selftest_startup_sysprof,
290 __init static int init_stack_trace(void)
292 return register_tracer(&stack_trace);
294 device_initcall(init_stack_trace);
296 #define MAX_LONG_DIGITS 22
299 sysprof_sample_read(struct file *filp, char __user *ubuf,
300 size_t cnt, loff_t *ppos)
302 char buf[MAX_LONG_DIGITS];
305 r = sprintf(buf, "%ld\n", nsecs_to_usecs(sample_period));
307 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
311 sysprof_sample_write(struct file *filp, const char __user *ubuf,
312 size_t cnt, loff_t *ppos)
314 char buf[MAX_LONG_DIGITS];
317 if (cnt > MAX_LONG_DIGITS-1)
318 cnt = MAX_LONG_DIGITS-1;
320 if (copy_from_user(&buf, ubuf, cnt))
325 val = simple_strtoul(buf, NULL, 10);
327 * Enforce a minimum sample period of 100 usecs:
332 mutex_lock(&sample_timer_lock);
334 sample_period = val * 1000;
335 start_stack_timers();
336 mutex_unlock(&sample_timer_lock);
341 static struct file_operations sysprof_sample_fops = {
342 .read = sysprof_sample_read,
343 .write = sysprof_sample_write,
346 void init_tracer_sysprof_debugfs(struct dentry *d_tracer)
348 struct dentry *entry;
350 entry = debugfs_create_file("sysprof_sample_period", 0644,
351 d_tracer, NULL, &sysprof_sample_fops);
354 pr_warning("Could not create debugfs 'dyn_ftrace_total_info' entry\n");