2 * ring buffer based function tracer
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
14 #include <linux/utsrelease.h>
15 #include <linux/kallsyms.h>
16 #include <linux/seq_file.h>
17 #include <linux/notifier.h>
18 #include <linux/debugfs.h>
19 #include <linux/pagemap.h>
20 #include <linux/hardirq.h>
21 #include <linux/linkage.h>
22 #include <linux/uaccess.h>
23 #include <linux/ftrace.h>
24 #include <linux/module.h>
25 #include <linux/percpu.h>
26 #include <linux/kdebug.h>
27 #include <linux/ctype.h>
28 #include <linux/init.h>
29 #include <linux/poll.h>
30 #include <linux/gfp.h>
32 #include <linux/kprobes.h>
33 #include <linux/writeback.h>
35 #include <linux/stacktrace.h>
36 #include <linux/ring_buffer.h>
37 #include <linux/irqflags.h>
41 #define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
43 unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX;
44 unsigned long __read_mostly tracing_thresh;
48 * Kill all tracing for good (never come back).
49 * It is initialized to 1 but will turn to zero if the initialization
50 * of the tracer is successful. But that is the only place that sets
53 int tracing_disabled = 1;
55 static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
57 static inline void ftrace_disable_cpu(void)
60 local_inc(&__get_cpu_var(ftrace_cpu_disabled));
63 static inline void ftrace_enable_cpu(void)
65 local_dec(&__get_cpu_var(ftrace_cpu_disabled));
69 static cpumask_t __read_mostly tracing_buffer_mask;
71 #define for_each_tracing_cpu(cpu) \
72 for_each_cpu_mask(cpu, tracing_buffer_mask)
75 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
77 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
78 * is set, then ftrace_dump is called. This will output the contents
79 * of the ftrace buffers to the console. This is very useful for
80 * capturing traces that lead to crashes and outputing it to a
83 * It is default off, but you can enable it with either specifying
84 * "ftrace_dump_on_oops" in the kernel command line, or setting
85 * /proc/sys/kernel/ftrace_dump_on_oops to true.
87 int ftrace_dump_on_oops;
89 static int tracing_set_tracer(char *buf);
91 static int __init set_ftrace(char *str)
93 tracing_set_tracer(str);
96 __setup("ftrace", set_ftrace);
98 static int __init set_ftrace_dump_on_oops(char *str)
100 ftrace_dump_on_oops = 1;
103 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
106 ns2usecs(cycle_t nsec)
113 cycle_t ftrace_now(int cpu)
115 u64 ts = ring_buffer_time_stamp(cpu);
116 ring_buffer_normalize_time_stamp(cpu, &ts);
121 * The global_trace is the descriptor that holds the tracing
122 * buffers for the live tracing. For each CPU, it contains
123 * a link list of pages that will store trace entries. The
124 * page descriptor of the pages in the memory is used to hold
125 * the link list by linking the lru item in the page descriptor
126 * to each of the pages in the buffer per CPU.
128 * For each active CPU there is a data field that holds the
129 * pages for the buffer for that CPU. Each CPU has the same number
130 * of pages allocated for its buffer.
132 static struct trace_array global_trace;
134 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
137 * The max_tr is used to snapshot the global_trace when a maximum
138 * latency is reached. Some tracers will use this to store a maximum
139 * trace while it continues examining live traces.
141 * The buffers for the max_tr are set up the same as the global_trace.
142 * When a snapshot is taken, the link list of the max_tr is swapped
143 * with the link list of the global_trace and the buffers are reset for
144 * the global_trace so the tracing can continue.
146 static struct trace_array max_tr;
148 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
150 /* tracer_enabled is used to toggle activation of a tracer */
151 static int tracer_enabled = 1;
154 * tracing_is_enabled - return tracer_enabled status
156 * This function is used by other tracers to know the status
157 * of the tracer_enabled flag. Tracers may use this function
158 * to know if it should enable their features when starting
159 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
161 int tracing_is_enabled(void)
163 return tracer_enabled;
166 /* function tracing enabled */
167 int ftrace_function_enabled;
170 * trace_buf_size is the size in bytes that is allocated
171 * for a buffer. Note, the number of bytes is always rounded
174 * This number is purposely set to a low number of 16384.
175 * If the dump on oops happens, it will be much appreciated
176 * to not have to wait for all that output. Anyway this can be
177 * boot time and run time configurable.
179 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
181 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
183 /* trace_types holds a link list of available tracers. */
184 static struct tracer *trace_types __read_mostly;
186 /* current_trace points to the tracer that is currently active */
187 static struct tracer *current_trace __read_mostly;
190 * max_tracer_type_len is used to simplify the allocating of
191 * buffers to read userspace tracer names. We keep track of
192 * the longest tracer name registered.
194 static int max_tracer_type_len;
197 * trace_types_lock is used to protect the trace_types list.
198 * This lock is also used to keep user access serialized.
199 * Accesses from userspace will grab this lock while userspace
200 * activities happen inside the kernel.
202 static DEFINE_MUTEX(trace_types_lock);
204 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
205 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
207 /* trace_flags holds iter_ctrl options */
208 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK;
211 * trace_wake_up - wake up tasks waiting for trace input
213 * Simply wakes up any task that is blocked on the trace_wait
214 * queue. These is used with trace_poll for tasks polling the trace.
216 void trace_wake_up(void)
219 * The runqueue_is_locked() can fail, but this is the best we
222 if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
223 wake_up(&trace_wait);
226 static int __init set_buf_size(char *str)
228 unsigned long buf_size;
233 ret = strict_strtoul(str, 0, &buf_size);
234 /* nr_entries can not be zero */
235 if (ret < 0 || buf_size == 0)
237 trace_buf_size = buf_size;
240 __setup("trace_buf_size=", set_buf_size);
242 unsigned long nsecs_to_usecs(unsigned long nsecs)
247 /* These must match the bit postions in trace_iterator_flags */
248 static const char *trace_options[] = {
265 * ftrace_max_lock is used to protect the swapping of buffers
266 * when taking a max snapshot. The buffers themselves are
267 * protected by per_cpu spinlocks. But the action of the swap
268 * needs its own lock.
270 * This is defined as a raw_spinlock_t in order to help
271 * with performance when lockdep debugging is enabled.
273 static raw_spinlock_t ftrace_max_lock =
274 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
277 * Copy the new maximum trace into the separate maximum-trace
278 * structure. (this way the maximum trace is permanently saved,
279 * for later retrieval via /debugfs/tracing/latency_trace)
282 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
284 struct trace_array_cpu *data = tr->data[cpu];
287 max_tr.time_start = data->preempt_timestamp;
289 data = max_tr.data[cpu];
290 data->saved_latency = tracing_max_latency;
292 memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
293 data->pid = tsk->pid;
294 data->uid = tsk->uid;
295 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
296 data->policy = tsk->policy;
297 data->rt_priority = tsk->rt_priority;
299 /* record this tasks comm */
300 tracing_record_cmdline(current);
304 * trace_seq_printf - sequence printing of trace information
305 * @s: trace sequence descriptor
306 * @fmt: printf format string
308 * The tracer may use either sequence operations or its own
309 * copy to user routines. To simplify formating of a trace
310 * trace_seq_printf is used to store strings into a special
311 * buffer (@s). Then the output may be either used by
312 * the sequencer or pulled into another buffer.
315 trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
317 int len = (PAGE_SIZE - 1) - s->len;
325 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
328 /* If we can't write it all, don't bother writing anything */
338 * trace_seq_puts - trace sequence printing of simple string
339 * @s: trace sequence descriptor
340 * @str: simple string to record
342 * The tracer may use either the sequence operations or its own
343 * copy to user routines. This function records a simple string
344 * into a special buffer (@s) for later retrieval by a sequencer
345 * or other mechanism.
348 trace_seq_puts(struct trace_seq *s, const char *str)
350 int len = strlen(str);
352 if (len > ((PAGE_SIZE - 1) - s->len))
355 memcpy(s->buffer + s->len, str, len);
362 trace_seq_putc(struct trace_seq *s, unsigned char c)
364 if (s->len >= (PAGE_SIZE - 1))
367 s->buffer[s->len++] = c;
373 trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
375 if (len > ((PAGE_SIZE - 1) - s->len))
378 memcpy(s->buffer + s->len, mem, len);
384 #define MAX_MEMHEX_BYTES 8
385 #define HEX_CHARS (MAX_MEMHEX_BYTES*2 + 1)
388 trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
390 unsigned char hex[HEX_CHARS];
391 unsigned char *data = mem;
395 for (i = 0, j = 0; i < len; i++) {
397 for (i = len-1, j = 0; i >= 0; i--) {
399 hex[j++] = hex_asc_hi(data[i]);
400 hex[j++] = hex_asc_lo(data[i]);
404 return trace_seq_putmem(s, hex, j);
408 trace_seq_reset(struct trace_seq *s)
414 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
419 if (s->len <= s->readpos)
422 len = s->len - s->readpos;
425 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
434 trace_print_seq(struct seq_file *m, struct trace_seq *s)
436 int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
439 seq_puts(m, s->buffer);
445 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
447 * @tsk: the task with the latency
448 * @cpu: The cpu that initiated the trace.
450 * Flip the buffers between the @tr and the max_tr and record information
451 * about which task was the cause of this latency.
454 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
456 struct ring_buffer *buf = tr->buffer;
458 WARN_ON_ONCE(!irqs_disabled());
459 __raw_spin_lock(&ftrace_max_lock);
461 tr->buffer = max_tr.buffer;
464 ftrace_disable_cpu();
465 ring_buffer_reset(tr->buffer);
468 __update_max_tr(tr, tsk, cpu);
469 __raw_spin_unlock(&ftrace_max_lock);
473 * update_max_tr_single - only copy one trace over, and reset the rest
475 * @tsk - task with the latency
476 * @cpu - the cpu of the buffer to copy.
478 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
481 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
485 WARN_ON_ONCE(!irqs_disabled());
486 __raw_spin_lock(&ftrace_max_lock);
488 ftrace_disable_cpu();
490 ring_buffer_reset(max_tr.buffer);
491 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
497 __update_max_tr(tr, tsk, cpu);
498 __raw_spin_unlock(&ftrace_max_lock);
502 * register_tracer - register a tracer with the ftrace system.
503 * @type - the plugin for the tracer
505 * Register a new plugin tracer.
507 int register_tracer(struct tracer *type)
514 pr_info("Tracer must have a name\n");
518 mutex_lock(&trace_types_lock);
519 for (t = trace_types; t; t = t->next) {
520 if (strcmp(type->name, t->name) == 0) {
522 pr_info("Trace %s already registered\n",
529 #ifdef CONFIG_FTRACE_STARTUP_TEST
530 if (type->selftest) {
531 struct tracer *saved_tracer = current_trace;
532 struct trace_array *tr = &global_trace;
535 * Run a selftest on this tracer.
536 * Here we reset the trace buffer, and set the current
537 * tracer to be this tracer. The tracer can then run some
538 * internal tracing to verify that everything is in order.
539 * If we fail, we do not register this tracer.
541 for_each_tracing_cpu(i) {
542 tracing_reset(tr, i);
544 current_trace = type;
545 /* the test is responsible for initializing and enabling */
546 pr_info("Testing tracer %s: ", type->name);
547 ret = type->selftest(type, tr);
548 /* the test is responsible for resetting too */
549 current_trace = saved_tracer;
551 printk(KERN_CONT "FAILED!\n");
554 /* Only reset on passing, to avoid touching corrupted buffers */
555 for_each_tracing_cpu(i) {
556 tracing_reset(tr, i);
558 printk(KERN_CONT "PASSED\n");
562 type->next = trace_types;
564 len = strlen(type->name);
565 if (len > max_tracer_type_len)
566 max_tracer_type_len = len;
569 mutex_unlock(&trace_types_lock);
574 void unregister_tracer(struct tracer *type)
579 mutex_lock(&trace_types_lock);
580 for (t = &trace_types; *t; t = &(*t)->next) {
584 pr_info("Trace %s not registered\n", type->name);
589 if (strlen(type->name) != max_tracer_type_len)
592 max_tracer_type_len = 0;
593 for (t = &trace_types; *t; t = &(*t)->next) {
594 len = strlen((*t)->name);
595 if (len > max_tracer_type_len)
596 max_tracer_type_len = len;
599 mutex_unlock(&trace_types_lock);
602 void tracing_reset(struct trace_array *tr, int cpu)
604 ftrace_disable_cpu();
605 ring_buffer_reset_cpu(tr->buffer, cpu);
609 #define SAVED_CMDLINES 128
610 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
611 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
612 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
613 static int cmdline_idx;
614 static DEFINE_SPINLOCK(trace_cmdline_lock);
616 /* temporary disable recording */
617 atomic_t trace_record_cmdline_disabled __read_mostly;
619 static void trace_init_cmdlines(void)
621 memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
622 memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
626 static int trace_stop_count;
627 static DEFINE_SPINLOCK(tracing_start_lock);
630 * tracing_start - quick start of the tracer
632 * If tracing is enabled but was stopped by tracing_stop,
633 * this will start the tracer back up.
635 void tracing_start(void)
637 struct ring_buffer *buffer;
640 if (tracing_disabled)
643 spin_lock_irqsave(&tracing_start_lock, flags);
644 if (--trace_stop_count)
647 if (trace_stop_count < 0) {
648 /* Someone screwed up their debugging */
650 trace_stop_count = 0;
655 buffer = global_trace.buffer;
657 ring_buffer_record_enable(buffer);
659 buffer = max_tr.buffer;
661 ring_buffer_record_enable(buffer);
665 spin_unlock_irqrestore(&tracing_start_lock, flags);
669 * tracing_stop - quick stop of the tracer
671 * Light weight way to stop tracing. Use in conjunction with
674 void tracing_stop(void)
676 struct ring_buffer *buffer;
680 spin_lock_irqsave(&tracing_start_lock, flags);
681 if (trace_stop_count++)
684 buffer = global_trace.buffer;
686 ring_buffer_record_disable(buffer);
688 buffer = max_tr.buffer;
690 ring_buffer_record_disable(buffer);
693 spin_unlock_irqrestore(&tracing_start_lock, flags);
696 void trace_stop_cmdline_recording(void);
698 static void trace_save_cmdline(struct task_struct *tsk)
703 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
707 * It's not the end of the world if we don't get
708 * the lock, but we also don't want to spin
709 * nor do we want to disable interrupts,
710 * so if we miss here, then better luck next time.
712 if (!spin_trylock(&trace_cmdline_lock))
715 idx = map_pid_to_cmdline[tsk->pid];
716 if (idx >= SAVED_CMDLINES) {
717 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
719 map = map_cmdline_to_pid[idx];
720 if (map <= PID_MAX_DEFAULT)
721 map_pid_to_cmdline[map] = (unsigned)-1;
723 map_pid_to_cmdline[tsk->pid] = idx;
728 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
730 spin_unlock(&trace_cmdline_lock);
733 static char *trace_find_cmdline(int pid)
735 char *cmdline = "<...>";
741 if (pid > PID_MAX_DEFAULT)
744 map = map_pid_to_cmdline[pid];
745 if (map >= SAVED_CMDLINES)
748 cmdline = saved_cmdlines[map];
754 void tracing_record_cmdline(struct task_struct *tsk)
756 if (atomic_read(&trace_record_cmdline_disabled))
759 trace_save_cmdline(tsk);
763 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
766 struct task_struct *tsk = current;
768 entry->preempt_count = pc & 0xff;
769 entry->pid = (tsk) ? tsk->pid : 0;
771 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
772 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
774 TRACE_FLAG_IRQS_NOSUPPORT |
776 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
777 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
778 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
782 trace_function(struct trace_array *tr, struct trace_array_cpu *data,
783 unsigned long ip, unsigned long parent_ip, unsigned long flags,
786 struct ring_buffer_event *event;
787 struct ftrace_entry *entry;
788 unsigned long irq_flags;
790 /* If we are reading the ring buffer, don't trace */
791 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
794 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
798 entry = ring_buffer_event_data(event);
799 tracing_generic_entry_update(&entry->ent, flags, pc);
800 entry->ent.type = TRACE_FN;
802 entry->parent_ip = parent_ip;
803 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
806 #ifdef CONFIG_FUNCTION_RET_TRACER
807 static void __trace_function_return(struct trace_array *tr,
808 struct trace_array_cpu *data,
809 struct ftrace_retfunc *trace,
813 struct ring_buffer_event *event;
814 struct ftrace_ret_entry *entry;
815 unsigned long irq_flags;
817 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
820 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry),
824 entry = ring_buffer_event_data(event);
825 tracing_generic_entry_update(&entry->ent, flags, pc);
826 entry->ent.type = TRACE_FN_RET;
827 entry->ip = trace->func;
828 entry->parent_ip = trace->ret;
829 entry->rettime = trace->rettime;
830 entry->calltime = trace->calltime;
831 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags);
836 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
837 unsigned long ip, unsigned long parent_ip, unsigned long flags,
840 if (likely(!atomic_read(&data->disabled)))
841 trace_function(tr, data, ip, parent_ip, flags, pc);
844 static void ftrace_trace_stack(struct trace_array *tr,
845 struct trace_array_cpu *data,
849 #ifdef CONFIG_STACKTRACE
850 struct ring_buffer_event *event;
851 struct stack_entry *entry;
852 struct stack_trace trace;
853 unsigned long irq_flags;
855 if (!(trace_flags & TRACE_ITER_STACKTRACE))
858 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
862 entry = ring_buffer_event_data(event);
863 tracing_generic_entry_update(&entry->ent, flags, pc);
864 entry->ent.type = TRACE_STACK;
866 memset(&entry->caller, 0, sizeof(entry->caller));
868 trace.nr_entries = 0;
869 trace.max_entries = FTRACE_STACK_ENTRIES;
871 trace.entries = entry->caller;
873 save_stack_trace(&trace);
874 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
878 void __trace_stack(struct trace_array *tr,
879 struct trace_array_cpu *data,
883 ftrace_trace_stack(tr, data, flags, skip, preempt_count());
887 ftrace_trace_special(void *__tr, void *__data,
888 unsigned long arg1, unsigned long arg2, unsigned long arg3,
891 struct ring_buffer_event *event;
892 struct trace_array_cpu *data = __data;
893 struct trace_array *tr = __tr;
894 struct special_entry *entry;
895 unsigned long irq_flags;
897 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
901 entry = ring_buffer_event_data(event);
902 tracing_generic_entry_update(&entry->ent, 0, pc);
903 entry->ent.type = TRACE_SPECIAL;
907 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
908 ftrace_trace_stack(tr, data, irq_flags, 4, pc);
914 __trace_special(void *__tr, void *__data,
915 unsigned long arg1, unsigned long arg2, unsigned long arg3)
917 ftrace_trace_special(__tr, __data, arg1, arg2, arg3, preempt_count());
921 tracing_sched_switch_trace(struct trace_array *tr,
922 struct trace_array_cpu *data,
923 struct task_struct *prev,
924 struct task_struct *next,
925 unsigned long flags, int pc)
927 struct ring_buffer_event *event;
928 struct ctx_switch_entry *entry;
929 unsigned long irq_flags;
931 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
935 entry = ring_buffer_event_data(event);
936 tracing_generic_entry_update(&entry->ent, flags, pc);
937 entry->ent.type = TRACE_CTX;
938 entry->prev_pid = prev->pid;
939 entry->prev_prio = prev->prio;
940 entry->prev_state = prev->state;
941 entry->next_pid = next->pid;
942 entry->next_prio = next->prio;
943 entry->next_state = next->state;
944 entry->next_cpu = task_cpu(next);
945 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
946 ftrace_trace_stack(tr, data, flags, 5, pc);
950 tracing_sched_wakeup_trace(struct trace_array *tr,
951 struct trace_array_cpu *data,
952 struct task_struct *wakee,
953 struct task_struct *curr,
954 unsigned long flags, int pc)
956 struct ring_buffer_event *event;
957 struct ctx_switch_entry *entry;
958 unsigned long irq_flags;
960 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
964 entry = ring_buffer_event_data(event);
965 tracing_generic_entry_update(&entry->ent, flags, pc);
966 entry->ent.type = TRACE_WAKE;
967 entry->prev_pid = curr->pid;
968 entry->prev_prio = curr->prio;
969 entry->prev_state = curr->state;
970 entry->next_pid = wakee->pid;
971 entry->next_prio = wakee->prio;
972 entry->next_state = wakee->state;
973 entry->next_cpu = task_cpu(wakee);
974 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
975 ftrace_trace_stack(tr, data, flags, 6, pc);
981 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
983 struct trace_array *tr = &global_trace;
984 struct trace_array_cpu *data;
989 if (tracing_disabled)
992 pc = preempt_count();
993 local_irq_save(flags);
994 cpu = raw_smp_processor_id();
995 data = tr->data[cpu];
997 if (likely(atomic_inc_return(&data->disabled) == 1))
998 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
1000 atomic_dec(&data->disabled);
1001 local_irq_restore(flags);
1004 #ifdef CONFIG_FUNCTION_TRACER
1006 function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
1008 struct trace_array *tr = &global_trace;
1009 struct trace_array_cpu *data;
1010 unsigned long flags;
1015 if (unlikely(!ftrace_function_enabled))
1018 pc = preempt_count();
1019 resched = ftrace_preempt_disable();
1020 local_save_flags(flags);
1021 cpu = raw_smp_processor_id();
1022 data = tr->data[cpu];
1023 disabled = atomic_inc_return(&data->disabled);
1025 if (likely(disabled == 1))
1026 trace_function(tr, data, ip, parent_ip, flags, pc);
1028 atomic_dec(&data->disabled);
1029 ftrace_preempt_enable(resched);
1033 function_trace_call(unsigned long ip, unsigned long parent_ip)
1035 struct trace_array *tr = &global_trace;
1036 struct trace_array_cpu *data;
1037 unsigned long flags;
1042 if (unlikely(!ftrace_function_enabled))
1046 * Need to use raw, since this must be called before the
1047 * recursive protection is performed.
1049 raw_local_irq_save(flags);
1050 cpu = raw_smp_processor_id();
1051 data = tr->data[cpu];
1052 disabled = atomic_inc_return(&data->disabled);
1054 if (likely(disabled == 1)) {
1055 pc = preempt_count();
1056 trace_function(tr, data, ip, parent_ip, flags, pc);
1059 atomic_dec(&data->disabled);
1060 raw_local_irq_restore(flags);
1063 #ifdef CONFIG_FUNCTION_RET_TRACER
1064 void trace_function_return(struct ftrace_retfunc *trace)
1066 struct trace_array *tr = &global_trace;
1067 struct trace_array_cpu *data;
1068 unsigned long flags;
1073 raw_local_irq_save(flags);
1074 cpu = raw_smp_processor_id();
1075 data = tr->data[cpu];
1076 disabled = atomic_inc_return(&data->disabled);
1077 if (likely(disabled == 1)) {
1078 pc = preempt_count();
1079 __trace_function_return(tr, data, trace, flags, pc);
1081 atomic_dec(&data->disabled);
1082 raw_local_irq_restore(flags);
1084 #endif /* CONFIG_FUNCTION_RET_TRACER */
1086 static struct ftrace_ops trace_ops __read_mostly =
1088 .func = function_trace_call,
1091 void tracing_start_function_trace(void)
1093 ftrace_function_enabled = 0;
1095 if (trace_flags & TRACE_ITER_PREEMPTONLY)
1096 trace_ops.func = function_trace_call_preempt_only;
1098 trace_ops.func = function_trace_call;
1100 register_ftrace_function(&trace_ops);
1101 ftrace_function_enabled = 1;
1104 void tracing_stop_function_trace(void)
1106 ftrace_function_enabled = 0;
1107 unregister_ftrace_function(&trace_ops);
1111 enum trace_file_type {
1112 TRACE_FILE_LAT_FMT = 1,
1115 static void trace_iterator_increment(struct trace_iterator *iter, int cpu)
1117 /* Don't allow ftrace to trace into the ring buffers */
1118 ftrace_disable_cpu();
1121 if (iter->buffer_iter[iter->cpu])
1122 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1124 ftrace_enable_cpu();
1127 static struct trace_entry *
1128 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
1130 struct ring_buffer_event *event;
1131 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1133 /* Don't allow ftrace to trace into the ring buffers */
1134 ftrace_disable_cpu();
1137 event = ring_buffer_iter_peek(buf_iter, ts);
1139 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1141 ftrace_enable_cpu();
1143 return event ? ring_buffer_event_data(event) : NULL;
1146 static struct trace_entry *
1147 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1149 struct ring_buffer *buffer = iter->tr->buffer;
1150 struct trace_entry *ent, *next = NULL;
1151 u64 next_ts = 0, ts;
1155 for_each_tracing_cpu(cpu) {
1157 if (ring_buffer_empty_cpu(buffer, cpu))
1160 ent = peek_next_entry(iter, cpu, &ts);
1163 * Pick the entry with the smallest timestamp:
1165 if (ent && (!next || ts < next_ts)) {
1173 *ent_cpu = next_cpu;
1181 /* Find the next real entry, without updating the iterator itself */
1182 static struct trace_entry *
1183 find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1185 return __find_next_entry(iter, ent_cpu, ent_ts);
1188 /* Find the next real entry, and increment the iterator to the next entry */
1189 static void *find_next_entry_inc(struct trace_iterator *iter)
1191 iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1194 trace_iterator_increment(iter, iter->cpu);
1196 return iter->ent ? iter : NULL;
1199 static void trace_consume(struct trace_iterator *iter)
1201 /* Don't allow ftrace to trace into the ring buffers */
1202 ftrace_disable_cpu();
1203 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1204 ftrace_enable_cpu();
1207 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1209 struct trace_iterator *iter = m->private;
1215 /* can't go backwards */
1220 ent = find_next_entry_inc(iter);
1224 while (ent && iter->idx < i)
1225 ent = find_next_entry_inc(iter);
1232 static void *s_start(struct seq_file *m, loff_t *pos)
1234 struct trace_iterator *iter = m->private;
1239 mutex_lock(&trace_types_lock);
1241 if (!current_trace || current_trace != iter->trace) {
1242 mutex_unlock(&trace_types_lock);
1246 atomic_inc(&trace_record_cmdline_disabled);
1248 if (*pos != iter->pos) {
1253 ftrace_disable_cpu();
1255 for_each_tracing_cpu(cpu) {
1256 ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1259 ftrace_enable_cpu();
1261 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1266 p = s_next(m, p, &l);
1272 static void s_stop(struct seq_file *m, void *p)
1274 atomic_dec(&trace_record_cmdline_disabled);
1275 mutex_unlock(&trace_types_lock);
1278 #ifdef CONFIG_KRETPROBES
1279 static inline const char *kretprobed(const char *name)
1281 static const char tramp_name[] = "kretprobe_trampoline";
1282 int size = sizeof(tramp_name);
1284 if (strncmp(tramp_name, name, size) == 0)
1285 return "[unknown/kretprobe'd]";
1289 static inline const char *kretprobed(const char *name)
1293 #endif /* CONFIG_KRETPROBES */
1296 seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1298 #ifdef CONFIG_KALLSYMS
1299 char str[KSYM_SYMBOL_LEN];
1302 kallsyms_lookup(address, NULL, NULL, NULL, str);
1304 name = kretprobed(str);
1306 return trace_seq_printf(s, fmt, name);
1312 seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1313 unsigned long address)
1315 #ifdef CONFIG_KALLSYMS
1316 char str[KSYM_SYMBOL_LEN];
1319 sprint_symbol(str, address);
1320 name = kretprobed(str);
1322 return trace_seq_printf(s, fmt, name);
1327 #ifndef CONFIG_64BIT
1328 # define IP_FMT "%08lx"
1330 # define IP_FMT "%016lx"
1334 seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1339 return trace_seq_printf(s, "0");
1341 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1342 ret = seq_print_sym_offset(s, "%s", ip);
1344 ret = seq_print_sym_short(s, "%s", ip);
1349 if (sym_flags & TRACE_ITER_SYM_ADDR)
1350 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1354 static void print_lat_help_header(struct seq_file *m)
1356 seq_puts(m, "# _------=> CPU# \n");
1357 seq_puts(m, "# / _-----=> irqs-off \n");
1358 seq_puts(m, "# | / _----=> need-resched \n");
1359 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1360 seq_puts(m, "# ||| / _--=> preempt-depth \n");
1361 seq_puts(m, "# |||| / \n");
1362 seq_puts(m, "# ||||| delay \n");
1363 seq_puts(m, "# cmd pid ||||| time | caller \n");
1364 seq_puts(m, "# \\ / ||||| \\ | / \n");
1367 static void print_func_help_header(struct seq_file *m)
1369 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1370 seq_puts(m, "# | | | | |\n");
1375 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1377 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1378 struct trace_array *tr = iter->tr;
1379 struct trace_array_cpu *data = tr->data[tr->cpu];
1380 struct tracer *type = current_trace;
1381 unsigned long total;
1382 unsigned long entries;
1383 const char *name = "preemption";
1388 entries = ring_buffer_entries(iter->tr->buffer);
1390 ring_buffer_overruns(iter->tr->buffer);
1392 seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1394 seq_puts(m, "-----------------------------------"
1395 "---------------------------------\n");
1396 seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1397 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1398 nsecs_to_usecs(data->saved_latency),
1402 #if defined(CONFIG_PREEMPT_NONE)
1404 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1406 #elif defined(CONFIG_PREEMPT)
1411 /* These are reserved for later use */
1414 seq_printf(m, " #P:%d)\n", num_online_cpus());
1418 seq_puts(m, " -----------------\n");
1419 seq_printf(m, " | task: %.16s-%d "
1420 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1421 data->comm, data->pid, data->uid, data->nice,
1422 data->policy, data->rt_priority);
1423 seq_puts(m, " -----------------\n");
1425 if (data->critical_start) {
1426 seq_puts(m, " => started at: ");
1427 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1428 trace_print_seq(m, &iter->seq);
1429 seq_puts(m, "\n => ended at: ");
1430 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1431 trace_print_seq(m, &iter->seq);
1439 lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1441 int hardirq, softirq;
1444 comm = trace_find_cmdline(entry->pid);
1446 trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1447 trace_seq_printf(s, "%3d", cpu);
1448 trace_seq_printf(s, "%c%c",
1449 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' :
1450 (entry->flags & TRACE_FLAG_IRQS_NOSUPPORT) ? 'X' : '.',
1451 ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1453 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1454 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1455 if (hardirq && softirq) {
1456 trace_seq_putc(s, 'H');
1459 trace_seq_putc(s, 'h');
1462 trace_seq_putc(s, 's');
1464 trace_seq_putc(s, '.');
1468 if (entry->preempt_count)
1469 trace_seq_printf(s, "%x", entry->preempt_count);
1471 trace_seq_puts(s, ".");
1474 unsigned long preempt_mark_thresh = 100;
1477 lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1478 unsigned long rel_usecs)
1480 trace_seq_printf(s, " %4lldus", abs_usecs);
1481 if (rel_usecs > preempt_mark_thresh)
1482 trace_seq_puts(s, "!: ");
1483 else if (rel_usecs > 1)
1484 trace_seq_puts(s, "+: ");
1486 trace_seq_puts(s, " : ");
1489 static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1492 * The message is supposed to contain an ending newline.
1493 * If the printing stops prematurely, try to add a newline of our own.
1495 void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1497 struct trace_entry *ent;
1498 struct trace_field_cont *cont;
1501 ent = peek_next_entry(iter, iter->cpu, NULL);
1502 if (!ent || ent->type != TRACE_CONT) {
1503 trace_seq_putc(s, '\n');
1508 cont = (struct trace_field_cont *)ent;
1510 ok = (trace_seq_printf(s, "%s", cont->buf) > 0);
1512 ftrace_disable_cpu();
1514 if (iter->buffer_iter[iter->cpu])
1515 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1517 ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
1519 ftrace_enable_cpu();
1521 ent = peek_next_entry(iter, iter->cpu, NULL);
1522 } while (ent && ent->type == TRACE_CONT);
1525 trace_seq_putc(s, '\n');
1528 static void test_cpu_buff_start(struct trace_iterator *iter)
1530 struct trace_seq *s = &iter->seq;
1532 if (cpu_isset(iter->cpu, iter->started))
1535 cpu_set(iter->cpu, iter->started);
1536 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
1539 static enum print_line_t
1540 print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1542 struct trace_seq *s = &iter->seq;
1543 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1544 struct trace_entry *next_entry;
1545 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1546 struct trace_entry *entry = iter->ent;
1547 unsigned long abs_usecs;
1548 unsigned long rel_usecs;
1555 if (entry->type == TRACE_CONT)
1556 return TRACE_TYPE_HANDLED;
1558 test_cpu_buff_start(iter);
1560 next_entry = find_next_entry(iter, NULL, &next_ts);
1563 rel_usecs = ns2usecs(next_ts - iter->ts);
1564 abs_usecs = ns2usecs(iter->ts - iter->tr->time_start);
1567 comm = trace_find_cmdline(entry->pid);
1568 trace_seq_printf(s, "%16s %5d %3d %d %08x %08x [%08lx]"
1569 " %ld.%03ldms (+%ld.%03ldms): ",
1571 entry->pid, cpu, entry->flags,
1572 entry->preempt_count, trace_idx,
1575 abs_usecs % 1000, rel_usecs/1000,
1578 lat_print_generic(s, entry, cpu);
1579 lat_print_timestamp(s, abs_usecs, rel_usecs);
1581 switch (entry->type) {
1583 struct ftrace_entry *field;
1585 trace_assign_type(field, entry);
1587 seq_print_ip_sym(s, field->ip, sym_flags);
1588 trace_seq_puts(s, " (");
1589 seq_print_ip_sym(s, field->parent_ip, sym_flags);
1590 trace_seq_puts(s, ")\n");
1595 struct ctx_switch_entry *field;
1597 trace_assign_type(field, entry);
1599 T = field->next_state < sizeof(state_to_char) ?
1600 state_to_char[field->next_state] : 'X';
1602 state = field->prev_state ?
1603 __ffs(field->prev_state) + 1 : 0;
1604 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1605 comm = trace_find_cmdline(field->next_pid);
1606 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1609 S, entry->type == TRACE_CTX ? "==>" : " +",
1616 case TRACE_SPECIAL: {
1617 struct special_entry *field;
1619 trace_assign_type(field, entry);
1621 trace_seq_printf(s, "# %ld %ld %ld\n",
1628 struct stack_entry *field;
1630 trace_assign_type(field, entry);
1632 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1634 trace_seq_puts(s, " <= ");
1635 seq_print_ip_sym(s, field->caller[i], sym_flags);
1637 trace_seq_puts(s, "\n");
1641 struct print_entry *field;
1643 trace_assign_type(field, entry);
1645 seq_print_ip_sym(s, field->ip, sym_flags);
1646 trace_seq_printf(s, ": %s", field->buf);
1647 if (entry->flags & TRACE_FLAG_CONT)
1648 trace_seq_print_cont(s, iter);
1652 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1654 return TRACE_TYPE_HANDLED;
1657 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1659 struct trace_seq *s = &iter->seq;
1660 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1661 struct trace_entry *entry;
1662 unsigned long usec_rem;
1663 unsigned long long t;
1672 if (entry->type == TRACE_CONT)
1673 return TRACE_TYPE_HANDLED;
1675 test_cpu_buff_start(iter);
1677 comm = trace_find_cmdline(iter->ent->pid);
1679 t = ns2usecs(iter->ts);
1680 usec_rem = do_div(t, 1000000ULL);
1681 secs = (unsigned long)t;
1683 ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1685 return TRACE_TYPE_PARTIAL_LINE;
1686 ret = trace_seq_printf(s, "[%03d] ", iter->cpu);
1688 return TRACE_TYPE_PARTIAL_LINE;
1689 ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1691 return TRACE_TYPE_PARTIAL_LINE;
1693 switch (entry->type) {
1695 struct ftrace_entry *field;
1697 trace_assign_type(field, entry);
1699 ret = seq_print_ip_sym(s, field->ip, sym_flags);
1701 return TRACE_TYPE_PARTIAL_LINE;
1702 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1704 ret = trace_seq_printf(s, " <-");
1706 return TRACE_TYPE_PARTIAL_LINE;
1707 ret = seq_print_ip_sym(s,
1711 return TRACE_TYPE_PARTIAL_LINE;
1713 ret = trace_seq_printf(s, "\n");
1715 return TRACE_TYPE_PARTIAL_LINE;
1720 struct ctx_switch_entry *field;
1722 trace_assign_type(field, entry);
1724 S = field->prev_state < sizeof(state_to_char) ?
1725 state_to_char[field->prev_state] : 'X';
1726 T = field->next_state < sizeof(state_to_char) ?
1727 state_to_char[field->next_state] : 'X';
1728 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1732 entry->type == TRACE_CTX ? "==>" : " +",
1738 return TRACE_TYPE_PARTIAL_LINE;
1741 case TRACE_SPECIAL: {
1742 struct special_entry *field;
1744 trace_assign_type(field, entry);
1746 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1751 return TRACE_TYPE_PARTIAL_LINE;
1755 struct stack_entry *field;
1757 trace_assign_type(field, entry);
1759 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1761 ret = trace_seq_puts(s, " <= ");
1763 return TRACE_TYPE_PARTIAL_LINE;
1765 ret = seq_print_ip_sym(s, field->caller[i],
1768 return TRACE_TYPE_PARTIAL_LINE;
1770 ret = trace_seq_puts(s, "\n");
1772 return TRACE_TYPE_PARTIAL_LINE;
1776 struct print_entry *field;
1778 trace_assign_type(field, entry);
1780 seq_print_ip_sym(s, field->ip, sym_flags);
1781 trace_seq_printf(s, ": %s", field->buf);
1782 if (entry->flags & TRACE_FLAG_CONT)
1783 trace_seq_print_cont(s, iter);
1786 case TRACE_FN_RET: {
1787 return print_return_function(iter);
1791 return TRACE_TYPE_HANDLED;
1794 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1796 struct trace_seq *s = &iter->seq;
1797 struct trace_entry *entry;
1803 if (entry->type == TRACE_CONT)
1804 return TRACE_TYPE_HANDLED;
1806 ret = trace_seq_printf(s, "%d %d %llu ",
1807 entry->pid, iter->cpu, iter->ts);
1809 return TRACE_TYPE_PARTIAL_LINE;
1811 switch (entry->type) {
1813 struct ftrace_entry *field;
1815 trace_assign_type(field, entry);
1817 ret = trace_seq_printf(s, "%x %x\n",
1821 return TRACE_TYPE_PARTIAL_LINE;
1826 struct ctx_switch_entry *field;
1828 trace_assign_type(field, entry);
1830 S = field->prev_state < sizeof(state_to_char) ?
1831 state_to_char[field->prev_state] : 'X';
1832 T = field->next_state < sizeof(state_to_char) ?
1833 state_to_char[field->next_state] : 'X';
1834 if (entry->type == TRACE_WAKE)
1836 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1845 return TRACE_TYPE_PARTIAL_LINE;
1850 struct special_entry *field;
1852 trace_assign_type(field, entry);
1854 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1859 return TRACE_TYPE_PARTIAL_LINE;
1863 struct print_entry *field;
1865 trace_assign_type(field, entry);
1867 trace_seq_printf(s, "# %lx %s", field->ip, field->buf);
1868 if (entry->flags & TRACE_FLAG_CONT)
1869 trace_seq_print_cont(s, iter);
1873 return TRACE_TYPE_HANDLED;
1876 #define SEQ_PUT_FIELD_RET(s, x) \
1878 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
1882 #define SEQ_PUT_HEX_FIELD_RET(s, x) \
1884 BUILD_BUG_ON(sizeof(x) > MAX_MEMHEX_BYTES); \
1885 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
1889 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1891 struct trace_seq *s = &iter->seq;
1892 unsigned char newline = '\n';
1893 struct trace_entry *entry;
1898 if (entry->type == TRACE_CONT)
1899 return TRACE_TYPE_HANDLED;
1901 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1902 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1903 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1905 switch (entry->type) {
1907 struct ftrace_entry *field;
1909 trace_assign_type(field, entry);
1911 SEQ_PUT_HEX_FIELD_RET(s, field->ip);
1912 SEQ_PUT_HEX_FIELD_RET(s, field->parent_ip);
1917 struct ctx_switch_entry *field;
1919 trace_assign_type(field, entry);
1921 S = field->prev_state < sizeof(state_to_char) ?
1922 state_to_char[field->prev_state] : 'X';
1923 T = field->next_state < sizeof(state_to_char) ?
1924 state_to_char[field->next_state] : 'X';
1925 if (entry->type == TRACE_WAKE)
1927 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
1928 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
1929 SEQ_PUT_HEX_FIELD_RET(s, S);
1930 SEQ_PUT_HEX_FIELD_RET(s, field->next_cpu);
1931 SEQ_PUT_HEX_FIELD_RET(s, field->next_pid);
1932 SEQ_PUT_HEX_FIELD_RET(s, field->next_prio);
1933 SEQ_PUT_HEX_FIELD_RET(s, T);
1938 struct special_entry *field;
1940 trace_assign_type(field, entry);
1942 SEQ_PUT_HEX_FIELD_RET(s, field->arg1);
1943 SEQ_PUT_HEX_FIELD_RET(s, field->arg2);
1944 SEQ_PUT_HEX_FIELD_RET(s, field->arg3);
1948 SEQ_PUT_FIELD_RET(s, newline);
1950 return TRACE_TYPE_HANDLED;
1953 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1955 struct trace_seq *s = &iter->seq;
1956 struct trace_entry *entry;
1960 if (entry->type == TRACE_CONT)
1961 return TRACE_TYPE_HANDLED;
1963 SEQ_PUT_FIELD_RET(s, entry->pid);
1964 SEQ_PUT_FIELD_RET(s, entry->cpu);
1965 SEQ_PUT_FIELD_RET(s, iter->ts);
1967 switch (entry->type) {
1969 struct ftrace_entry *field;
1971 trace_assign_type(field, entry);
1973 SEQ_PUT_FIELD_RET(s, field->ip);
1974 SEQ_PUT_FIELD_RET(s, field->parent_ip);
1978 struct ctx_switch_entry *field;
1980 trace_assign_type(field, entry);
1982 SEQ_PUT_FIELD_RET(s, field->prev_pid);
1983 SEQ_PUT_FIELD_RET(s, field->prev_prio);
1984 SEQ_PUT_FIELD_RET(s, field->prev_state);
1985 SEQ_PUT_FIELD_RET(s, field->next_pid);
1986 SEQ_PUT_FIELD_RET(s, field->next_prio);
1987 SEQ_PUT_FIELD_RET(s, field->next_state);
1992 struct special_entry *field;
1994 trace_assign_type(field, entry);
1996 SEQ_PUT_FIELD_RET(s, field->arg1);
1997 SEQ_PUT_FIELD_RET(s, field->arg2);
1998 SEQ_PUT_FIELD_RET(s, field->arg3);
2005 static int trace_empty(struct trace_iterator *iter)
2009 for_each_tracing_cpu(cpu) {
2010 if (iter->buffer_iter[cpu]) {
2011 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2014 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2022 static enum print_line_t print_trace_line(struct trace_iterator *iter)
2024 enum print_line_t ret;
2026 if (iter->trace && iter->trace->print_line) {
2027 ret = iter->trace->print_line(iter);
2028 if (ret != TRACE_TYPE_UNHANDLED)
2032 if (trace_flags & TRACE_ITER_BIN)
2033 return print_bin_fmt(iter);
2035 if (trace_flags & TRACE_ITER_HEX)
2036 return print_hex_fmt(iter);
2038 if (trace_flags & TRACE_ITER_RAW)
2039 return print_raw_fmt(iter);
2041 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2042 return print_lat_fmt(iter, iter->idx, iter->cpu);
2044 return print_trace_fmt(iter);
2047 static int s_show(struct seq_file *m, void *v)
2049 struct trace_iterator *iter = v;
2051 if (iter->ent == NULL) {
2053 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2056 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2057 /* print nothing if the buffers are empty */
2058 if (trace_empty(iter))
2060 print_trace_header(m, iter);
2061 if (!(trace_flags & TRACE_ITER_VERBOSE))
2062 print_lat_help_header(m);
2064 if (!(trace_flags & TRACE_ITER_VERBOSE))
2065 print_func_help_header(m);
2068 print_trace_line(iter);
2069 trace_print_seq(m, &iter->seq);
2075 static struct seq_operations tracer_seq_ops = {
2082 static struct trace_iterator *
2083 __tracing_open(struct inode *inode, struct file *file, int *ret)
2085 struct trace_iterator *iter;
2089 if (tracing_disabled) {
2094 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2100 mutex_lock(&trace_types_lock);
2101 if (current_trace && current_trace->print_max)
2104 iter->tr = inode->i_private;
2105 iter->trace = current_trace;
2108 for_each_tracing_cpu(cpu) {
2110 iter->buffer_iter[cpu] =
2111 ring_buffer_read_start(iter->tr->buffer, cpu);
2113 if (!iter->buffer_iter[cpu])
2117 /* TODO stop tracer */
2118 *ret = seq_open(file, &tracer_seq_ops);
2122 m = file->private_data;
2125 /* stop the trace while dumping */
2128 if (iter->trace && iter->trace->open)
2129 iter->trace->open(iter);
2131 mutex_unlock(&trace_types_lock);
2137 for_each_tracing_cpu(cpu) {
2138 if (iter->buffer_iter[cpu])
2139 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2141 mutex_unlock(&trace_types_lock);
2143 return ERR_PTR(-ENOMEM);
2146 int tracing_open_generic(struct inode *inode, struct file *filp)
2148 if (tracing_disabled)
2151 filp->private_data = inode->i_private;
2155 int tracing_release(struct inode *inode, struct file *file)
2157 struct seq_file *m = (struct seq_file *)file->private_data;
2158 struct trace_iterator *iter = m->private;
2161 mutex_lock(&trace_types_lock);
2162 for_each_tracing_cpu(cpu) {
2163 if (iter->buffer_iter[cpu])
2164 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2167 if (iter->trace && iter->trace->close)
2168 iter->trace->close(iter);
2170 /* reenable tracing if it was previously enabled */
2172 mutex_unlock(&trace_types_lock);
2174 seq_release(inode, file);
2179 static int tracing_open(struct inode *inode, struct file *file)
2183 __tracing_open(inode, file, &ret);
2188 static int tracing_lt_open(struct inode *inode, struct file *file)
2190 struct trace_iterator *iter;
2193 iter = __tracing_open(inode, file, &ret);
2196 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2203 t_next(struct seq_file *m, void *v, loff_t *pos)
2205 struct tracer *t = m->private;
2217 static void *t_start(struct seq_file *m, loff_t *pos)
2219 struct tracer *t = m->private;
2222 mutex_lock(&trace_types_lock);
2223 for (; t && l < *pos; t = t_next(m, t, &l))
2229 static void t_stop(struct seq_file *m, void *p)
2231 mutex_unlock(&trace_types_lock);
2234 static int t_show(struct seq_file *m, void *v)
2236 struct tracer *t = v;
2241 seq_printf(m, "%s", t->name);
2250 static struct seq_operations show_traces_seq_ops = {
2257 static int show_traces_open(struct inode *inode, struct file *file)
2261 if (tracing_disabled)
2264 ret = seq_open(file, &show_traces_seq_ops);
2266 struct seq_file *m = file->private_data;
2267 m->private = trace_types;
2273 static struct file_operations tracing_fops = {
2274 .open = tracing_open,
2276 .llseek = seq_lseek,
2277 .release = tracing_release,
2280 static struct file_operations tracing_lt_fops = {
2281 .open = tracing_lt_open,
2283 .llseek = seq_lseek,
2284 .release = tracing_release,
2287 static struct file_operations show_traces_fops = {
2288 .open = show_traces_open,
2290 .release = seq_release,
2294 * Only trace on a CPU if the bitmask is set:
2296 static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2299 * When tracing/tracing_cpu_mask is modified then this holds
2300 * the new bitmask we are about to install:
2302 static cpumask_t tracing_cpumask_new;
2305 * The tracer itself will not take this lock, but still we want
2306 * to provide a consistent cpumask to user-space:
2308 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2311 * Temporary storage for the character representation of the
2312 * CPU bitmask (and one more byte for the newline):
2314 static char mask_str[NR_CPUS + 1];
2317 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2318 size_t count, loff_t *ppos)
2322 mutex_lock(&tracing_cpumask_update_lock);
2324 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2325 if (count - len < 2) {
2329 len += sprintf(mask_str + len, "\n");
2330 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2333 mutex_unlock(&tracing_cpumask_update_lock);
2339 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2340 size_t count, loff_t *ppos)
2344 mutex_lock(&tracing_cpumask_update_lock);
2345 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2349 raw_local_irq_disable();
2350 __raw_spin_lock(&ftrace_max_lock);
2351 for_each_tracing_cpu(cpu) {
2353 * Increase/decrease the disabled counter if we are
2354 * about to flip a bit in the cpumask:
2356 if (cpu_isset(cpu, tracing_cpumask) &&
2357 !cpu_isset(cpu, tracing_cpumask_new)) {
2358 atomic_inc(&global_trace.data[cpu]->disabled);
2360 if (!cpu_isset(cpu, tracing_cpumask) &&
2361 cpu_isset(cpu, tracing_cpumask_new)) {
2362 atomic_dec(&global_trace.data[cpu]->disabled);
2365 __raw_spin_unlock(&ftrace_max_lock);
2366 raw_local_irq_enable();
2368 tracing_cpumask = tracing_cpumask_new;
2370 mutex_unlock(&tracing_cpumask_update_lock);
2375 mutex_unlock(&tracing_cpumask_update_lock);
2380 static struct file_operations tracing_cpumask_fops = {
2381 .open = tracing_open_generic,
2382 .read = tracing_cpumask_read,
2383 .write = tracing_cpumask_write,
2387 tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2388 size_t cnt, loff_t *ppos)
2395 /* calulate max size */
2396 for (i = 0; trace_options[i]; i++) {
2397 len += strlen(trace_options[i]);
2398 len += 3; /* "no" and space */
2401 /* +2 for \n and \0 */
2402 buf = kmalloc(len + 2, GFP_KERNEL);
2406 for (i = 0; trace_options[i]; i++) {
2407 if (trace_flags & (1 << i))
2408 r += sprintf(buf + r, "%s ", trace_options[i]);
2410 r += sprintf(buf + r, "no%s ", trace_options[i]);
2413 r += sprintf(buf + r, "\n");
2414 WARN_ON(r >= len + 2);
2416 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2424 tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2425 size_t cnt, loff_t *ppos)
2432 if (cnt >= sizeof(buf))
2435 if (copy_from_user(&buf, ubuf, cnt))
2440 if (strncmp(buf, "no", 2) == 0) {
2445 for (i = 0; trace_options[i]; i++) {
2446 int len = strlen(trace_options[i]);
2448 if (strncmp(cmp, trace_options[i], len) == 0) {
2450 trace_flags &= ~(1 << i);
2452 trace_flags |= (1 << i);
2457 * If no option could be set, return an error:
2459 if (!trace_options[i])
2467 static struct file_operations tracing_iter_fops = {
2468 .open = tracing_open_generic,
2469 .read = tracing_iter_ctrl_read,
2470 .write = tracing_iter_ctrl_write,
2473 static const char readme_msg[] =
2474 "tracing mini-HOWTO:\n\n"
2476 "# mount -t debugfs nodev /debug\n\n"
2477 "# cat /debug/tracing/available_tracers\n"
2478 "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2479 "# cat /debug/tracing/current_tracer\n"
2481 "# echo sched_switch > /debug/tracing/current_tracer\n"
2482 "# cat /debug/tracing/current_tracer\n"
2484 "# cat /debug/tracing/iter_ctrl\n"
2485 "noprint-parent nosym-offset nosym-addr noverbose\n"
2486 "# echo print-parent > /debug/tracing/iter_ctrl\n"
2487 "# echo 1 > /debug/tracing/tracing_enabled\n"
2488 "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2489 "echo 0 > /debug/tracing/tracing_enabled\n"
2493 tracing_readme_read(struct file *filp, char __user *ubuf,
2494 size_t cnt, loff_t *ppos)
2496 return simple_read_from_buffer(ubuf, cnt, ppos,
2497 readme_msg, strlen(readme_msg));
2500 static struct file_operations tracing_readme_fops = {
2501 .open = tracing_open_generic,
2502 .read = tracing_readme_read,
2506 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2507 size_t cnt, loff_t *ppos)
2512 r = sprintf(buf, "%u\n", tracer_enabled);
2513 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2517 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2518 size_t cnt, loff_t *ppos)
2520 struct trace_array *tr = filp->private_data;
2525 if (cnt >= sizeof(buf))
2528 if (copy_from_user(&buf, ubuf, cnt))
2533 ret = strict_strtoul(buf, 10, &val);
2539 mutex_lock(&trace_types_lock);
2540 if (tracer_enabled ^ val) {
2543 if (current_trace->start)
2544 current_trace->start(tr);
2549 if (current_trace->stop)
2550 current_trace->stop(tr);
2553 mutex_unlock(&trace_types_lock);
2561 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2562 size_t cnt, loff_t *ppos)
2564 char buf[max_tracer_type_len+2];
2567 mutex_lock(&trace_types_lock);
2569 r = sprintf(buf, "%s\n", current_trace->name);
2571 r = sprintf(buf, "\n");
2572 mutex_unlock(&trace_types_lock);
2574 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2577 static int tracing_set_tracer(char *buf)
2579 struct trace_array *tr = &global_trace;
2583 mutex_lock(&trace_types_lock);
2584 for (t = trace_types; t; t = t->next) {
2585 if (strcmp(t->name, buf) == 0)
2592 if (t == current_trace)
2595 if (current_trace && current_trace->reset)
2596 current_trace->reset(tr);
2603 mutex_unlock(&trace_types_lock);
2609 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2610 size_t cnt, loff_t *ppos)
2612 char buf[max_tracer_type_len+1];
2616 if (cnt > max_tracer_type_len)
2617 cnt = max_tracer_type_len;
2619 if (copy_from_user(&buf, ubuf, cnt))
2624 /* strip ending whitespace. */
2625 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2628 ret = tracing_set_tracer(buf);
2639 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2640 size_t cnt, loff_t *ppos)
2642 unsigned long *ptr = filp->private_data;
2646 r = snprintf(buf, sizeof(buf), "%ld\n",
2647 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2648 if (r > sizeof(buf))
2650 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2654 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2655 size_t cnt, loff_t *ppos)
2657 long *ptr = filp->private_data;
2662 if (cnt >= sizeof(buf))
2665 if (copy_from_user(&buf, ubuf, cnt))
2670 ret = strict_strtoul(buf, 10, &val);
2679 static atomic_t tracing_reader;
2681 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2683 struct trace_iterator *iter;
2685 if (tracing_disabled)
2688 /* We only allow for reader of the pipe */
2689 if (atomic_inc_return(&tracing_reader) != 1) {
2690 atomic_dec(&tracing_reader);
2694 /* create a buffer to store the information to pass to userspace */
2695 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2699 mutex_lock(&trace_types_lock);
2701 /* trace pipe does not show start of buffer */
2702 cpus_setall(iter->started);
2704 iter->tr = &global_trace;
2705 iter->trace = current_trace;
2706 filp->private_data = iter;
2708 if (iter->trace->pipe_open)
2709 iter->trace->pipe_open(iter);
2710 mutex_unlock(&trace_types_lock);
2715 static int tracing_release_pipe(struct inode *inode, struct file *file)
2717 struct trace_iterator *iter = file->private_data;
2720 atomic_dec(&tracing_reader);
2726 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2728 struct trace_iterator *iter = filp->private_data;
2730 if (trace_flags & TRACE_ITER_BLOCK) {
2732 * Always select as readable when in blocking mode
2734 return POLLIN | POLLRDNORM;
2736 if (!trace_empty(iter))
2737 return POLLIN | POLLRDNORM;
2738 poll_wait(filp, &trace_wait, poll_table);
2739 if (!trace_empty(iter))
2740 return POLLIN | POLLRDNORM;
2750 tracing_read_pipe(struct file *filp, char __user *ubuf,
2751 size_t cnt, loff_t *ppos)
2753 struct trace_iterator *iter = filp->private_data;
2756 /* return any leftover data */
2757 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2761 trace_seq_reset(&iter->seq);
2763 mutex_lock(&trace_types_lock);
2764 if (iter->trace->read) {
2765 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2772 while (trace_empty(iter)) {
2774 if ((filp->f_flags & O_NONBLOCK)) {
2780 * This is a make-shift waitqueue. The reason we don't use
2781 * an actual wait queue is because:
2782 * 1) we only ever have one waiter
2783 * 2) the tracing, traces all functions, we don't want
2784 * the overhead of calling wake_up and friends
2785 * (and tracing them too)
2786 * Anyway, this is really very primitive wakeup.
2788 set_current_state(TASK_INTERRUPTIBLE);
2789 iter->tr->waiter = current;
2791 mutex_unlock(&trace_types_lock);
2793 /* sleep for 100 msecs, and try again. */
2794 schedule_timeout(HZ/10);
2796 mutex_lock(&trace_types_lock);
2798 iter->tr->waiter = NULL;
2800 if (signal_pending(current)) {
2805 if (iter->trace != current_trace)
2809 * We block until we read something and tracing is disabled.
2810 * We still block if tracing is disabled, but we have never
2811 * read anything. This allows a user to cat this file, and
2812 * then enable tracing. But after we have read something,
2813 * we give an EOF when tracing is again disabled.
2815 * iter->pos will be 0 if we haven't read anything.
2817 if (!tracer_enabled && iter->pos)
2823 /* stop when tracing is finished */
2824 if (trace_empty(iter))
2827 if (cnt >= PAGE_SIZE)
2828 cnt = PAGE_SIZE - 1;
2830 /* reset all but tr, trace, and overruns */
2831 memset(&iter->seq, 0,
2832 sizeof(struct trace_iterator) -
2833 offsetof(struct trace_iterator, seq));
2836 while (find_next_entry_inc(iter) != NULL) {
2837 enum print_line_t ret;
2838 int len = iter->seq.len;
2840 ret = print_trace_line(iter);
2841 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2842 /* don't print partial lines */
2843 iter->seq.len = len;
2847 trace_consume(iter);
2849 if (iter->seq.len >= cnt)
2853 /* Now copy what we have to the user */
2854 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2855 if (iter->seq.readpos >= iter->seq.len)
2856 trace_seq_reset(&iter->seq);
2859 * If there was nothing to send to user, inspite of consuming trace
2860 * entries, go back to wait for more entries.
2866 mutex_unlock(&trace_types_lock);
2872 tracing_entries_read(struct file *filp, char __user *ubuf,
2873 size_t cnt, loff_t *ppos)
2875 struct trace_array *tr = filp->private_data;
2879 r = sprintf(buf, "%lu\n", tr->entries);
2880 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2884 tracing_entries_write(struct file *filp, const char __user *ubuf,
2885 size_t cnt, loff_t *ppos)
2891 if (cnt >= sizeof(buf))
2894 if (copy_from_user(&buf, ubuf, cnt))
2899 ret = strict_strtoul(buf, 10, &val);
2903 /* must have at least 1 entry */
2907 mutex_lock(&trace_types_lock);
2911 /* disable all cpu buffers */
2912 for_each_tracing_cpu(cpu) {
2913 if (global_trace.data[cpu])
2914 atomic_inc(&global_trace.data[cpu]->disabled);
2915 if (max_tr.data[cpu])
2916 atomic_inc(&max_tr.data[cpu]->disabled);
2919 if (val != global_trace.entries) {
2920 ret = ring_buffer_resize(global_trace.buffer, val);
2926 ret = ring_buffer_resize(max_tr.buffer, val);
2930 r = ring_buffer_resize(global_trace.buffer,
2931 global_trace.entries);
2933 /* AARGH! We are left with different
2934 * size max buffer!!!! */
2936 tracing_disabled = 1;
2941 global_trace.entries = val;
2946 /* If check pages failed, return ENOMEM */
2947 if (tracing_disabled)
2950 for_each_tracing_cpu(cpu) {
2951 if (global_trace.data[cpu])
2952 atomic_dec(&global_trace.data[cpu]->disabled);
2953 if (max_tr.data[cpu])
2954 atomic_dec(&max_tr.data[cpu]->disabled);
2958 max_tr.entries = global_trace.entries;
2959 mutex_unlock(&trace_types_lock);
2964 static int mark_printk(const char *fmt, ...)
2968 va_start(args, fmt);
2969 ret = trace_vprintk(0, fmt, args);
2975 tracing_mark_write(struct file *filp, const char __user *ubuf,
2976 size_t cnt, loff_t *fpos)
2981 if (tracing_disabled)
2984 if (cnt > TRACE_BUF_SIZE)
2985 cnt = TRACE_BUF_SIZE;
2987 buf = kmalloc(cnt + 1, GFP_KERNEL);
2991 if (copy_from_user(buf, ubuf, cnt)) {
2996 /* Cut from the first nil or newline. */
2998 end = strchr(buf, '\n');
3002 cnt = mark_printk("%s\n", buf);
3009 static struct file_operations tracing_max_lat_fops = {
3010 .open = tracing_open_generic,
3011 .read = tracing_max_lat_read,
3012 .write = tracing_max_lat_write,
3015 static struct file_operations tracing_ctrl_fops = {
3016 .open = tracing_open_generic,
3017 .read = tracing_ctrl_read,
3018 .write = tracing_ctrl_write,
3021 static struct file_operations set_tracer_fops = {
3022 .open = tracing_open_generic,
3023 .read = tracing_set_trace_read,
3024 .write = tracing_set_trace_write,
3027 static struct file_operations tracing_pipe_fops = {
3028 .open = tracing_open_pipe,
3029 .poll = tracing_poll_pipe,
3030 .read = tracing_read_pipe,
3031 .release = tracing_release_pipe,
3034 static struct file_operations tracing_entries_fops = {
3035 .open = tracing_open_generic,
3036 .read = tracing_entries_read,
3037 .write = tracing_entries_write,
3040 static struct file_operations tracing_mark_fops = {
3041 .open = tracing_open_generic,
3042 .write = tracing_mark_write,
3045 #ifdef CONFIG_DYNAMIC_FTRACE
3047 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3053 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3054 size_t cnt, loff_t *ppos)
3056 static char ftrace_dyn_info_buffer[1024];
3057 static DEFINE_MUTEX(dyn_info_mutex);
3058 unsigned long *p = filp->private_data;
3059 char *buf = ftrace_dyn_info_buffer;
3060 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3063 mutex_lock(&dyn_info_mutex);
3064 r = sprintf(buf, "%ld ", *p);
3066 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3069 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3071 mutex_unlock(&dyn_info_mutex);
3076 static struct file_operations tracing_dyn_info_fops = {
3077 .open = tracing_open_generic,
3078 .read = tracing_read_dyn_info,
3082 static struct dentry *d_tracer;
3084 struct dentry *tracing_init_dentry(void)
3091 d_tracer = debugfs_create_dir("tracing", NULL);
3093 if (!d_tracer && !once) {
3095 pr_warning("Could not create debugfs directory 'tracing'\n");
3102 #ifdef CONFIG_FTRACE_SELFTEST
3103 /* Let selftest have access to static functions in this file */
3104 #include "trace_selftest.c"
3107 static __init int tracer_init_debugfs(void)
3109 struct dentry *d_tracer;
3110 struct dentry *entry;
3112 d_tracer = tracing_init_dentry();
3114 entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
3115 &global_trace, &tracing_ctrl_fops);
3117 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
3119 entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
3120 NULL, &tracing_iter_fops);
3122 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
3124 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
3125 NULL, &tracing_cpumask_fops);
3127 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
3129 entry = debugfs_create_file("latency_trace", 0444, d_tracer,
3130 &global_trace, &tracing_lt_fops);
3132 pr_warning("Could not create debugfs 'latency_trace' entry\n");
3134 entry = debugfs_create_file("trace", 0444, d_tracer,
3135 &global_trace, &tracing_fops);
3137 pr_warning("Could not create debugfs 'trace' entry\n");
3139 entry = debugfs_create_file("available_tracers", 0444, d_tracer,
3140 &global_trace, &show_traces_fops);
3142 pr_warning("Could not create debugfs 'available_tracers' entry\n");
3144 entry = debugfs_create_file("current_tracer", 0444, d_tracer,
3145 &global_trace, &set_tracer_fops);
3147 pr_warning("Could not create debugfs 'current_tracer' entry\n");
3149 entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
3150 &tracing_max_latency,
3151 &tracing_max_lat_fops);
3153 pr_warning("Could not create debugfs "
3154 "'tracing_max_latency' entry\n");
3156 entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
3157 &tracing_thresh, &tracing_max_lat_fops);
3159 pr_warning("Could not create debugfs "
3160 "'tracing_thresh' entry\n");
3161 entry = debugfs_create_file("README", 0644, d_tracer,
3162 NULL, &tracing_readme_fops);
3164 pr_warning("Could not create debugfs 'README' entry\n");
3166 entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
3167 NULL, &tracing_pipe_fops);
3169 pr_warning("Could not create debugfs "
3170 "'trace_pipe' entry\n");
3172 entry = debugfs_create_file("trace_entries", 0644, d_tracer,
3173 &global_trace, &tracing_entries_fops);
3175 pr_warning("Could not create debugfs "
3176 "'trace_entries' entry\n");
3178 entry = debugfs_create_file("trace_marker", 0220, d_tracer,
3179 NULL, &tracing_mark_fops);
3181 pr_warning("Could not create debugfs "
3182 "'trace_marker' entry\n");
3184 #ifdef CONFIG_DYNAMIC_FTRACE
3185 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
3186 &ftrace_update_tot_cnt,
3187 &tracing_dyn_info_fops);
3189 pr_warning("Could not create debugfs "
3190 "'dyn_ftrace_total_info' entry\n");
3192 #ifdef CONFIG_SYSPROF_TRACER
3193 init_tracer_sysprof_debugfs(d_tracer);
3198 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3200 static DEFINE_SPINLOCK(trace_buf_lock);
3201 static char trace_buf[TRACE_BUF_SIZE];
3203 struct ring_buffer_event *event;
3204 struct trace_array *tr = &global_trace;
3205 struct trace_array_cpu *data;
3206 struct print_entry *entry;
3207 unsigned long flags, irq_flags;
3208 int cpu, len = 0, size, pc;
3210 if (tracing_disabled)
3213 pc = preempt_count();
3214 preempt_disable_notrace();
3215 cpu = raw_smp_processor_id();
3216 data = tr->data[cpu];
3218 if (unlikely(atomic_read(&data->disabled)))
3221 spin_lock_irqsave(&trace_buf_lock, flags);
3222 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
3224 len = min(len, TRACE_BUF_SIZE-1);
3227 size = sizeof(*entry) + len + 1;
3228 event = ring_buffer_lock_reserve(tr->buffer, size, &irq_flags);
3231 entry = ring_buffer_event_data(event);
3232 tracing_generic_entry_update(&entry->ent, flags, pc);
3233 entry->ent.type = TRACE_PRINT;
3236 memcpy(&entry->buf, trace_buf, len);
3237 entry->buf[len] = 0;
3238 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
3241 spin_unlock_irqrestore(&trace_buf_lock, flags);
3244 preempt_enable_notrace();
3248 EXPORT_SYMBOL_GPL(trace_vprintk);
3250 int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3255 if (!(trace_flags & TRACE_ITER_PRINTK))
3259 ret = trace_vprintk(ip, fmt, ap);
3263 EXPORT_SYMBOL_GPL(__ftrace_printk);
3265 static int trace_panic_handler(struct notifier_block *this,
3266 unsigned long event, void *unused)
3268 if (ftrace_dump_on_oops)
3273 static struct notifier_block trace_panic_notifier = {
3274 .notifier_call = trace_panic_handler,
3276 .priority = 150 /* priority: INT_MAX >= x >= 0 */
3279 static int trace_die_handler(struct notifier_block *self,
3285 if (ftrace_dump_on_oops)
3294 static struct notifier_block trace_die_notifier = {
3295 .notifier_call = trace_die_handler,
3300 * printk is set to max of 1024, we really don't need it that big.
3301 * Nothing should be printing 1000 characters anyway.
3303 #define TRACE_MAX_PRINT 1000
3306 * Define here KERN_TRACE so that we have one place to modify
3307 * it if we decide to change what log level the ftrace dump
3310 #define KERN_TRACE KERN_INFO
3313 trace_printk_seq(struct trace_seq *s)
3315 /* Probably should print a warning here. */
3319 /* should be zero ended, but we are paranoid. */
3320 s->buffer[s->len] = 0;
3322 printk(KERN_TRACE "%s", s->buffer);
3327 void ftrace_dump(void)
3329 static DEFINE_SPINLOCK(ftrace_dump_lock);
3330 /* use static because iter can be a bit big for the stack */
3331 static struct trace_iterator iter;
3332 static cpumask_t mask;
3333 static int dump_ran;
3334 unsigned long flags;
3338 spin_lock_irqsave(&ftrace_dump_lock, flags);
3344 /* No turning back! */
3347 for_each_tracing_cpu(cpu) {
3348 atomic_inc(&global_trace.data[cpu]->disabled);
3351 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3353 iter.tr = &global_trace;
3354 iter.trace = current_trace;
3357 * We need to stop all tracing on all CPUS to read the
3358 * the next buffer. This is a bit expensive, but is
3359 * not done often. We fill all what we can read,
3360 * and then release the locks again.
3365 while (!trace_empty(&iter)) {
3368 printk(KERN_TRACE "---------------------------------\n");
3372 /* reset all but tr, trace, and overruns */
3373 memset(&iter.seq, 0,
3374 sizeof(struct trace_iterator) -
3375 offsetof(struct trace_iterator, seq));
3376 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3379 if (find_next_entry_inc(&iter) != NULL) {
3380 print_trace_line(&iter);
3381 trace_consume(&iter);
3384 trace_printk_seq(&iter.seq);
3388 printk(KERN_TRACE " (ftrace buffer empty)\n");
3390 printk(KERN_TRACE "---------------------------------\n");
3393 spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3396 __init static int tracer_alloc_buffers(void)
3398 struct trace_array_cpu *data;
3401 /* TODO: make the number of buffers hot pluggable with CPUS */
3402 tracing_buffer_mask = cpu_possible_map;
3404 global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3405 TRACE_BUFFER_FLAGS);
3406 if (!global_trace.buffer) {
3407 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3411 global_trace.entries = ring_buffer_size(global_trace.buffer);
3413 #ifdef CONFIG_TRACER_MAX_TRACE
3414 max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3415 TRACE_BUFFER_FLAGS);
3416 if (!max_tr.buffer) {
3417 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3419 ring_buffer_free(global_trace.buffer);
3422 max_tr.entries = ring_buffer_size(max_tr.buffer);
3423 WARN_ON(max_tr.entries != global_trace.entries);
3426 /* Allocate the first page for all buffers */
3427 for_each_tracing_cpu(i) {
3428 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3429 max_tr.data[i] = &per_cpu(max_data, i);
3432 trace_init_cmdlines();
3434 register_tracer(&nop_trace);
3435 #ifdef CONFIG_BOOT_TRACER
3436 register_tracer(&boot_tracer);
3437 current_trace = &boot_tracer;
3438 current_trace->init(&global_trace);
3440 current_trace = &nop_trace;
3443 /* All seems OK, enable tracing */
3444 tracing_disabled = 0;
3446 atomic_notifier_chain_register(&panic_notifier_list,
3447 &trace_panic_notifier);
3449 register_die_notifier(&trace_die_notifier);
3453 early_initcall(tracer_alloc_buffers);
3454 fs_initcall(tracer_init_debugfs);