4 * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
7 #include <linux/module.h>
9 #include <linux/debugfs.h>
10 #include <linux/kallsyms.h>
11 #include <linux/uaccess.h>
12 #include <linux/marker.h>
13 #include <linux/ftrace.h>
17 static struct trace_array *ctx_trace;
18 static int __read_mostly tracer_enabled;
19 static atomic_t sched_ref;
22 sched_switch_func(void *private, void *__rq, struct task_struct *prev,
23 struct task_struct *next)
25 struct trace_array **ptr = private;
26 struct trace_array *tr = *ptr;
27 struct trace_array_cpu *data;
35 local_irq_save(flags);
36 cpu = raw_smp_processor_id();
38 disabled = atomic_inc_return(&data->disabled);
40 if (likely(disabled == 1))
41 tracing_sched_switch_trace(tr, data, prev, next, flags);
43 atomic_dec(&data->disabled);
44 local_irq_restore(flags);
48 sched_switch_callback(void *probe_data, void *call_data,
49 const char *format, va_list *args)
51 struct task_struct *prev;
52 struct task_struct *next;
55 if (!atomic_read(&sched_ref))
58 /* skip prev_pid %d next_pid %d prev_state %ld */
59 (void)va_arg(*args, int);
60 (void)va_arg(*args, int);
61 (void)va_arg(*args, long);
62 __rq = va_arg(*args, typeof(__rq));
63 prev = va_arg(*args, typeof(prev));
64 next = va_arg(*args, typeof(next));
66 tracing_record_cmdline(prev);
69 * If tracer_switch_func only points to the local
70 * switch func, it still needs the ptr passed to it.
72 sched_switch_func(probe_data, __rq, prev, next);
76 wakeup_func(void *private, void *__rq, struct task_struct *wakee, struct
79 struct trace_array **ptr = private;
80 struct trace_array *tr = *ptr;
81 struct trace_array_cpu *data;
89 tracing_record_cmdline(curr);
91 local_irq_save(flags);
92 cpu = raw_smp_processor_id();
94 disabled = atomic_inc_return(&data->disabled);
96 if (likely(disabled == 1))
97 tracing_sched_wakeup_trace(tr, data, wakee, curr, flags);
99 atomic_dec(&data->disabled);
100 local_irq_restore(flags);
104 wake_up_callback(void *probe_data, void *call_data,
105 const char *format, va_list *args)
107 struct task_struct *curr;
108 struct task_struct *task;
111 if (likely(!tracer_enabled))
114 /* Skip pid %d state %ld */
115 (void)va_arg(*args, int);
116 (void)va_arg(*args, long);
117 /* now get the meat: "rq %p task %p rq->curr %p" */
118 __rq = va_arg(*args, typeof(__rq));
119 task = va_arg(*args, typeof(task));
120 curr = va_arg(*args, typeof(curr));
122 tracing_record_cmdline(task);
123 tracing_record_cmdline(curr);
125 wakeup_func(probe_data, __rq, task, curr);
129 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
131 struct trace_array *tr = ctx_trace;
132 struct trace_array_cpu *data;
140 local_irq_save(flags);
141 cpu = raw_smp_processor_id();
142 data = tr->data[cpu];
143 disabled = atomic_inc_return(&data->disabled);
145 if (likely(disabled == 1))
146 __trace_special(tr, data, arg1, arg2, arg3);
148 atomic_dec(&data->disabled);
149 local_irq_restore(flags);
152 static void sched_switch_reset(struct trace_array *tr)
156 tr->time_start = ftrace_now(tr->cpu);
158 for_each_online_cpu(cpu)
159 tracing_reset(tr->data[cpu]);
162 static int tracing_sched_register(void)
166 ret = marker_probe_register("kernel_sched_wakeup",
167 "pid %d state %ld ## rq %p task %p rq->curr %p",
171 pr_info("wakeup trace: Couldn't add marker"
172 " probe to kernel_sched_wakeup\n");
176 ret = marker_probe_register("kernel_sched_wakeup_new",
177 "pid %d state %ld ## rq %p task %p rq->curr %p",
181 pr_info("wakeup trace: Couldn't add marker"
182 " probe to kernel_sched_wakeup_new\n");
186 ret = marker_probe_register("kernel_sched_schedule",
187 "prev_pid %d next_pid %d prev_state %ld "
188 "## rq %p prev %p next %p",
189 sched_switch_callback,
192 pr_info("sched trace: Couldn't add marker"
193 " probe to kernel_sched_schedule\n");
194 goto fail_deprobe_wake_new;
198 fail_deprobe_wake_new:
199 marker_probe_unregister("kernel_sched_wakeup_new",
203 marker_probe_unregister("kernel_sched_wakeup",
209 static void tracing_sched_unregister(void)
211 marker_probe_unregister("kernel_sched_schedule",
212 sched_switch_callback,
214 marker_probe_unregister("kernel_sched_wakeup_new",
217 marker_probe_unregister("kernel_sched_wakeup",
222 void tracing_start_sched_switch(void)
226 ref = atomic_inc_return(&sched_ref);
228 tracing_sched_register();
231 void tracing_stop_sched_switch(void)
235 ref = atomic_dec_and_test(&sched_ref);
237 tracing_sched_unregister();
240 static void start_sched_trace(struct trace_array *tr)
242 sched_switch_reset(tr);
243 atomic_inc(&trace_record_cmdline_enabled);
245 tracing_start_sched_switch();
248 static void stop_sched_trace(struct trace_array *tr)
250 tracing_stop_sched_switch();
251 atomic_dec(&trace_record_cmdline_enabled);
255 static void sched_switch_trace_init(struct trace_array *tr)
260 start_sched_trace(tr);
263 static void sched_switch_trace_reset(struct trace_array *tr)
266 stop_sched_trace(tr);
269 static void sched_switch_trace_ctrl_update(struct trace_array *tr)
271 /* When starting a new trace, reset the buffers */
273 start_sched_trace(tr);
275 stop_sched_trace(tr);
278 static struct tracer sched_switch_trace __read_mostly =
280 .name = "sched_switch",
281 .init = sched_switch_trace_init,
282 .reset = sched_switch_trace_reset,
283 .ctrl_update = sched_switch_trace_ctrl_update,
284 #ifdef CONFIG_FTRACE_SELFTEST
285 .selftest = trace_selftest_startup_sched_switch,
289 __init static int init_sched_switch_trace(void)
293 if (atomic_read(&sched_ref))
294 ret = tracing_sched_register();
296 pr_info("error registering scheduler trace\n");
299 return register_tracer(&sched_switch_trace);
301 device_initcall(init_sched_switch_trace);