2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled = 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
99 cpumask_var_t __read_mostly tracing_buffer_mask;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning;
122 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
128 static bool allocate_snapshot;
130 static int __init set_cmdline_ftrace(char *str)
132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133 default_bootup_tracer = bootup_tracer_buf;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = true;
138 __setup("ftrace=", set_cmdline_ftrace);
140 static int __init set_ftrace_dump_on_oops(char *str)
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
156 static int __init stop_trace_on_warning(char *str)
158 __disable_trace_on_warning = 1;
161 __setup("traceoff_on_warning=", stop_trace_on_warning);
163 static int __init boot_alloc_snapshot(char *str)
165 allocate_snapshot = true;
166 /* We also need the main ring buffer expanded */
167 ring_buffer_expanded = true;
170 __setup("alloc_snapshot", boot_alloc_snapshot);
173 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
174 static char *trace_boot_options __initdata;
176 static int __init set_trace_boot_options(char *str)
178 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
179 trace_boot_options = trace_boot_options_buf;
182 __setup("trace_options=", set_trace_boot_options);
184 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
185 static char *trace_boot_clock __initdata;
187 static int __init set_trace_boot_clock(char *str)
189 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
190 trace_boot_clock = trace_boot_clock_buf;
193 __setup("trace_clock=", set_trace_boot_clock);
196 unsigned long long ns2usecs(cycle_t nsec)
204 * The global_trace is the descriptor that holds the tracing
205 * buffers for the live tracing. For each CPU, it contains
206 * a link list of pages that will store trace entries. The
207 * page descriptor of the pages in the memory is used to hold
208 * the link list by linking the lru item in the page descriptor
209 * to each of the pages in the buffer per CPU.
211 * For each active CPU there is a data field that holds the
212 * pages for the buffer for that CPU. Each CPU has the same number
213 * of pages allocated for its buffer.
215 static struct trace_array global_trace;
217 LIST_HEAD(ftrace_trace_arrays);
219 int trace_array_get(struct trace_array *this_tr)
221 struct trace_array *tr;
224 mutex_lock(&trace_types_lock);
225 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
232 mutex_unlock(&trace_types_lock);
237 static void __trace_array_put(struct trace_array *this_tr)
239 WARN_ON(!this_tr->ref);
243 void trace_array_put(struct trace_array *this_tr)
245 mutex_lock(&trace_types_lock);
246 __trace_array_put(this_tr);
247 mutex_unlock(&trace_types_lock);
250 int filter_check_discard(struct ftrace_event_file *file, void *rec,
251 struct ring_buffer *buffer,
252 struct ring_buffer_event *event)
254 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
255 !filter_match_preds(file->filter, rec)) {
256 ring_buffer_discard_commit(buffer, event);
262 EXPORT_SYMBOL_GPL(filter_check_discard);
264 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
265 struct ring_buffer *buffer,
266 struct ring_buffer_event *event)
268 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
269 !filter_match_preds(call->filter, rec)) {
270 ring_buffer_discard_commit(buffer, event);
276 EXPORT_SYMBOL_GPL(call_filter_check_discard);
278 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
282 /* Early boot up does not have a buffer yet */
284 return trace_clock_local();
286 ts = ring_buffer_time_stamp(buf->buffer, cpu);
287 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
292 cycle_t ftrace_now(int cpu)
294 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
298 * tracing_is_enabled - Show if global_trace has been disabled
300 * Shows if the global trace has been enabled or not. It uses the
301 * mirror flag "buffer_disabled" to be used in fast paths such as for
302 * the irqsoff tracer. But it may be inaccurate due to races. If you
303 * need to know the accurate state, use tracing_is_on() which is a little
304 * slower, but accurate.
306 int tracing_is_enabled(void)
309 * For quick access (irqsoff uses this in fast path), just
310 * return the mirror variable of the state of the ring buffer.
311 * It's a little racy, but we don't really care.
314 return !global_trace.buffer_disabled;
318 * trace_buf_size is the size in bytes that is allocated
319 * for a buffer. Note, the number of bytes is always rounded
322 * This number is purposely set to a low number of 16384.
323 * If the dump on oops happens, it will be much appreciated
324 * to not have to wait for all that output. Anyway this can be
325 * boot time and run time configurable.
327 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
329 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
331 /* trace_types holds a link list of available tracers. */
332 static struct tracer *trace_types __read_mostly;
335 * trace_types_lock is used to protect the trace_types list.
337 DEFINE_MUTEX(trace_types_lock);
340 * serialize the access of the ring buffer
342 * ring buffer serializes readers, but it is low level protection.
343 * The validity of the events (which returns by ring_buffer_peek() ..etc)
344 * are not protected by ring buffer.
346 * The content of events may become garbage if we allow other process consumes
347 * these events concurrently:
348 * A) the page of the consumed events may become a normal page
349 * (not reader page) in ring buffer, and this page will be rewrited
350 * by events producer.
351 * B) The page of the consumed events may become a page for splice_read,
352 * and this page will be returned to system.
354 * These primitives allow multi process access to different cpu ring buffer
357 * These primitives don't distinguish read-only and read-consume access.
358 * Multi read-only access are also serialized.
362 static DECLARE_RWSEM(all_cpu_access_lock);
363 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
365 static inline void trace_access_lock(int cpu)
367 if (cpu == RING_BUFFER_ALL_CPUS) {
368 /* gain it for accessing the whole ring buffer. */
369 down_write(&all_cpu_access_lock);
371 /* gain it for accessing a cpu ring buffer. */
373 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
374 down_read(&all_cpu_access_lock);
376 /* Secondly block other access to this @cpu ring buffer. */
377 mutex_lock(&per_cpu(cpu_access_lock, cpu));
381 static inline void trace_access_unlock(int cpu)
383 if (cpu == RING_BUFFER_ALL_CPUS) {
384 up_write(&all_cpu_access_lock);
386 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
387 up_read(&all_cpu_access_lock);
391 static inline void trace_access_lock_init(void)
395 for_each_possible_cpu(cpu)
396 mutex_init(&per_cpu(cpu_access_lock, cpu));
401 static DEFINE_MUTEX(access_lock);
403 static inline void trace_access_lock(int cpu)
406 mutex_lock(&access_lock);
409 static inline void trace_access_unlock(int cpu)
412 mutex_unlock(&access_lock);
415 static inline void trace_access_lock_init(void)
421 /* trace_flags holds trace_options default values */
422 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
423 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
424 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
425 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
427 static void tracer_tracing_on(struct trace_array *tr)
429 if (tr->trace_buffer.buffer)
430 ring_buffer_record_on(tr->trace_buffer.buffer);
432 * This flag is looked at when buffers haven't been allocated
433 * yet, or by some tracers (like irqsoff), that just want to
434 * know if the ring buffer has been disabled, but it can handle
435 * races of where it gets disabled but we still do a record.
436 * As the check is in the fast path of the tracers, it is more
437 * important to be fast than accurate.
439 tr->buffer_disabled = 0;
440 /* Make the flag seen by readers */
445 * tracing_on - enable tracing buffers
447 * This function enables tracing buffers that may have been
448 * disabled with tracing_off.
450 void tracing_on(void)
452 tracer_tracing_on(&global_trace);
454 EXPORT_SYMBOL_GPL(tracing_on);
457 * __trace_puts - write a constant string into the trace buffer.
458 * @ip: The address of the caller
459 * @str: The constant string to write
460 * @size: The size of the string.
462 int __trace_puts(unsigned long ip, const char *str, int size)
464 struct ring_buffer_event *event;
465 struct ring_buffer *buffer;
466 struct print_entry *entry;
467 unsigned long irq_flags;
470 if (unlikely(tracing_selftest_running || tracing_disabled))
473 alloc = sizeof(*entry) + size + 2; /* possible \n added */
475 local_save_flags(irq_flags);
476 buffer = global_trace.trace_buffer.buffer;
477 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
478 irq_flags, preempt_count());
482 entry = ring_buffer_event_data(event);
485 memcpy(&entry->buf, str, size);
487 /* Add a newline if necessary */
488 if (entry->buf[size - 1] != '\n') {
489 entry->buf[size] = '\n';
490 entry->buf[size + 1] = '\0';
492 entry->buf[size] = '\0';
494 __buffer_unlock_commit(buffer, event);
498 EXPORT_SYMBOL_GPL(__trace_puts);
501 * __trace_bputs - write the pointer to a constant string into trace buffer
502 * @ip: The address of the caller
503 * @str: The constant string to write to the buffer to
505 int __trace_bputs(unsigned long ip, const char *str)
507 struct ring_buffer_event *event;
508 struct ring_buffer *buffer;
509 struct bputs_entry *entry;
510 unsigned long irq_flags;
511 int size = sizeof(struct bputs_entry);
513 if (unlikely(tracing_selftest_running || tracing_disabled))
516 local_save_flags(irq_flags);
517 buffer = global_trace.trace_buffer.buffer;
518 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
519 irq_flags, preempt_count());
523 entry = ring_buffer_event_data(event);
527 __buffer_unlock_commit(buffer, event);
531 EXPORT_SYMBOL_GPL(__trace_bputs);
533 #ifdef CONFIG_TRACER_SNAPSHOT
535 * trace_snapshot - take a snapshot of the current buffer.
537 * This causes a swap between the snapshot buffer and the current live
538 * tracing buffer. You can use this to take snapshots of the live
539 * trace when some condition is triggered, but continue to trace.
541 * Note, make sure to allocate the snapshot with either
542 * a tracing_snapshot_alloc(), or by doing it manually
543 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
545 * If the snapshot buffer is not allocated, it will stop tracing.
546 * Basically making a permanent snapshot.
548 void tracing_snapshot(void)
550 struct trace_array *tr = &global_trace;
551 struct tracer *tracer = tr->current_trace;
555 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
556 internal_trace_puts("*** snapshot is being ignored ***\n");
560 if (!tr->allocated_snapshot) {
561 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
562 internal_trace_puts("*** stopping trace here! ***\n");
567 /* Note, snapshot can not be used when the tracer uses it */
568 if (tracer->use_max_tr) {
569 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
570 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
574 local_irq_save(flags);
575 update_max_tr(tr, current, smp_processor_id());
576 local_irq_restore(flags);
578 EXPORT_SYMBOL_GPL(tracing_snapshot);
580 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
581 struct trace_buffer *size_buf, int cpu_id);
582 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
584 static int alloc_snapshot(struct trace_array *tr)
588 if (!tr->allocated_snapshot) {
590 /* allocate spare buffer */
591 ret = resize_buffer_duplicate_size(&tr->max_buffer,
592 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
596 tr->allocated_snapshot = true;
602 static void free_snapshot(struct trace_array *tr)
605 * We don't free the ring buffer. instead, resize it because
606 * The max_tr ring buffer has some state (e.g. ring->clock) and
607 * we want preserve it.
609 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
610 set_buffer_entries(&tr->max_buffer, 1);
611 tracing_reset_online_cpus(&tr->max_buffer);
612 tr->allocated_snapshot = false;
616 * tracing_alloc_snapshot - allocate snapshot buffer.
618 * This only allocates the snapshot buffer if it isn't already
619 * allocated - it doesn't also take a snapshot.
621 * This is meant to be used in cases where the snapshot buffer needs
622 * to be set up for events that can't sleep but need to be able to
623 * trigger a snapshot.
625 int tracing_alloc_snapshot(void)
627 struct trace_array *tr = &global_trace;
630 ret = alloc_snapshot(tr);
635 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
638 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
640 * This is similar to trace_snapshot(), but it will allocate the
641 * snapshot buffer if it isn't already allocated. Use this only
642 * where it is safe to sleep, as the allocation may sleep.
644 * This causes a swap between the snapshot buffer and the current live
645 * tracing buffer. You can use this to take snapshots of the live
646 * trace when some condition is triggered, but continue to trace.
648 void tracing_snapshot_alloc(void)
652 ret = tracing_alloc_snapshot();
658 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
660 void tracing_snapshot(void)
662 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
664 EXPORT_SYMBOL_GPL(tracing_snapshot);
665 int tracing_alloc_snapshot(void)
667 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
670 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
671 void tracing_snapshot_alloc(void)
676 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
677 #endif /* CONFIG_TRACER_SNAPSHOT */
679 static void tracer_tracing_off(struct trace_array *tr)
681 if (tr->trace_buffer.buffer)
682 ring_buffer_record_off(tr->trace_buffer.buffer);
684 * This flag is looked at when buffers haven't been allocated
685 * yet, or by some tracers (like irqsoff), that just want to
686 * know if the ring buffer has been disabled, but it can handle
687 * races of where it gets disabled but we still do a record.
688 * As the check is in the fast path of the tracers, it is more
689 * important to be fast than accurate.
691 tr->buffer_disabled = 1;
692 /* Make the flag seen by readers */
697 * tracing_off - turn off tracing buffers
699 * This function stops the tracing buffers from recording data.
700 * It does not disable any overhead the tracers themselves may
701 * be causing. This function simply causes all recording to
702 * the ring buffers to fail.
704 void tracing_off(void)
706 tracer_tracing_off(&global_trace);
708 EXPORT_SYMBOL_GPL(tracing_off);
710 void disable_trace_on_warning(void)
712 if (__disable_trace_on_warning)
717 * tracer_tracing_is_on - show real state of ring buffer enabled
718 * @tr : the trace array to know if ring buffer is enabled
720 * Shows real state of the ring buffer if it is enabled or not.
722 static int tracer_tracing_is_on(struct trace_array *tr)
724 if (tr->trace_buffer.buffer)
725 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
726 return !tr->buffer_disabled;
730 * tracing_is_on - show state of ring buffers enabled
732 int tracing_is_on(void)
734 return tracer_tracing_is_on(&global_trace);
736 EXPORT_SYMBOL_GPL(tracing_is_on);
738 static int __init set_buf_size(char *str)
740 unsigned long buf_size;
744 buf_size = memparse(str, &str);
745 /* nr_entries can not be zero */
748 trace_buf_size = buf_size;
751 __setup("trace_buf_size=", set_buf_size);
753 static int __init set_tracing_thresh(char *str)
755 unsigned long threshold;
760 ret = kstrtoul(str, 0, &threshold);
763 tracing_thresh = threshold * 1000;
766 __setup("tracing_thresh=", set_tracing_thresh);
768 unsigned long nsecs_to_usecs(unsigned long nsecs)
773 /* These must match the bit postions in trace_iterator_flags */
774 static const char *trace_options[] = {
807 int in_ns; /* is this clock in nanoseconds? */
809 { trace_clock_local, "local", 1 },
810 { trace_clock_global, "global", 1 },
811 { trace_clock_counter, "counter", 0 },
812 { trace_clock_jiffies, "uptime", 1 },
813 { trace_clock, "perf", 1 },
818 * trace_parser_get_init - gets the buffer for trace parser
820 int trace_parser_get_init(struct trace_parser *parser, int size)
822 memset(parser, 0, sizeof(*parser));
824 parser->buffer = kmalloc(size, GFP_KERNEL);
833 * trace_parser_put - frees the buffer for trace parser
835 void trace_parser_put(struct trace_parser *parser)
837 kfree(parser->buffer);
841 * trace_get_user - reads the user input string separated by space
842 * (matched by isspace(ch))
844 * For each string found the 'struct trace_parser' is updated,
845 * and the function returns.
847 * Returns number of bytes read.
849 * See kernel/trace/trace.h for 'struct trace_parser' details.
851 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
852 size_t cnt, loff_t *ppos)
859 trace_parser_clear(parser);
861 ret = get_user(ch, ubuf++);
869 * The parser is not finished with the last write,
870 * continue reading the user input without skipping spaces.
873 /* skip white space */
874 while (cnt && isspace(ch)) {
875 ret = get_user(ch, ubuf++);
882 /* only spaces were written */
892 /* read the non-space input */
893 while (cnt && !isspace(ch)) {
894 if (parser->idx < parser->size - 1)
895 parser->buffer[parser->idx++] = ch;
900 ret = get_user(ch, ubuf++);
907 /* We either got finished input or we have to wait for another call. */
909 parser->buffer[parser->idx] = 0;
910 parser->cont = false;
911 } else if (parser->idx < parser->size - 1) {
913 parser->buffer[parser->idx++] = ch;
926 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
930 if (s->len <= s->readpos)
933 len = s->len - s->readpos;
936 memcpy(buf, s->buffer + s->readpos, cnt);
942 unsigned long __read_mostly tracing_thresh;
944 #ifdef CONFIG_TRACER_MAX_TRACE
946 * Copy the new maximum trace into the separate maximum-trace
947 * structure. (this way the maximum trace is permanently saved,
948 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
951 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
953 struct trace_buffer *trace_buf = &tr->trace_buffer;
954 struct trace_buffer *max_buf = &tr->max_buffer;
955 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
956 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
959 max_buf->time_start = data->preempt_timestamp;
961 max_data->saved_latency = tr->max_latency;
962 max_data->critical_start = data->critical_start;
963 max_data->critical_end = data->critical_end;
965 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
966 max_data->pid = tsk->pid;
968 * If tsk == current, then use current_uid(), as that does not use
969 * RCU. The irq tracer can be called out of RCU scope.
972 max_data->uid = current_uid();
974 max_data->uid = task_uid(tsk);
976 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
977 max_data->policy = tsk->policy;
978 max_data->rt_priority = tsk->rt_priority;
980 /* record this tasks comm */
981 tracing_record_cmdline(tsk);
985 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
987 * @tsk: the task with the latency
988 * @cpu: The cpu that initiated the trace.
990 * Flip the buffers between the @tr and the max_tr and record information
991 * about which task was the cause of this latency.
994 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
996 struct ring_buffer *buf;
1001 WARN_ON_ONCE(!irqs_disabled());
1003 if (!tr->allocated_snapshot) {
1004 /* Only the nop tracer should hit this when disabling */
1005 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1009 arch_spin_lock(&tr->max_lock);
1011 buf = tr->trace_buffer.buffer;
1012 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1013 tr->max_buffer.buffer = buf;
1015 __update_max_tr(tr, tsk, cpu);
1016 arch_spin_unlock(&tr->max_lock);
1020 * update_max_tr_single - only copy one trace over, and reset the rest
1022 * @tsk - task with the latency
1023 * @cpu - the cpu of the buffer to copy.
1025 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1028 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1035 WARN_ON_ONCE(!irqs_disabled());
1036 if (!tr->allocated_snapshot) {
1037 /* Only the nop tracer should hit this when disabling */
1038 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1042 arch_spin_lock(&tr->max_lock);
1044 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1046 if (ret == -EBUSY) {
1048 * We failed to swap the buffer due to a commit taking
1049 * place on this CPU. We fail to record, but we reset
1050 * the max trace buffer (no one writes directly to it)
1051 * and flag that it failed.
1053 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1054 "Failed to swap buffers due to commit in progress\n");
1057 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1059 __update_max_tr(tr, tsk, cpu);
1060 arch_spin_unlock(&tr->max_lock);
1062 #endif /* CONFIG_TRACER_MAX_TRACE */
1064 static int wait_on_pipe(struct trace_iterator *iter)
1066 /* Iterators are static, they should be filled or empty */
1067 if (trace_buffer_iter(iter, iter->cpu_file))
1070 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1073 #ifdef CONFIG_FTRACE_STARTUP_TEST
1074 static int run_tracer_selftest(struct tracer *type)
1076 struct trace_array *tr = &global_trace;
1077 struct tracer *saved_tracer = tr->current_trace;
1080 if (!type->selftest || tracing_selftest_disabled)
1084 * Run a selftest on this tracer.
1085 * Here we reset the trace buffer, and set the current
1086 * tracer to be this tracer. The tracer can then run some
1087 * internal tracing to verify that everything is in order.
1088 * If we fail, we do not register this tracer.
1090 tracing_reset_online_cpus(&tr->trace_buffer);
1092 tr->current_trace = type;
1094 #ifdef CONFIG_TRACER_MAX_TRACE
1095 if (type->use_max_tr) {
1096 /* If we expanded the buffers, make sure the max is expanded too */
1097 if (ring_buffer_expanded)
1098 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1099 RING_BUFFER_ALL_CPUS);
1100 tr->allocated_snapshot = true;
1104 /* the test is responsible for initializing and enabling */
1105 pr_info("Testing tracer %s: ", type->name);
1106 ret = type->selftest(type, tr);
1107 /* the test is responsible for resetting too */
1108 tr->current_trace = saved_tracer;
1110 printk(KERN_CONT "FAILED!\n");
1111 /* Add the warning after printing 'FAILED' */
1115 /* Only reset on passing, to avoid touching corrupted buffers */
1116 tracing_reset_online_cpus(&tr->trace_buffer);
1118 #ifdef CONFIG_TRACER_MAX_TRACE
1119 if (type->use_max_tr) {
1120 tr->allocated_snapshot = false;
1122 /* Shrink the max buffer again */
1123 if (ring_buffer_expanded)
1124 ring_buffer_resize(tr->max_buffer.buffer, 1,
1125 RING_BUFFER_ALL_CPUS);
1129 printk(KERN_CONT "PASSED\n");
1133 static inline int run_tracer_selftest(struct tracer *type)
1137 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1140 * register_tracer - register a tracer with the ftrace system.
1141 * @type - the plugin for the tracer
1143 * Register a new plugin tracer.
1145 int register_tracer(struct tracer *type)
1151 pr_info("Tracer must have a name\n");
1155 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1156 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1160 mutex_lock(&trace_types_lock);
1162 tracing_selftest_running = true;
1164 for (t = trace_types; t; t = t->next) {
1165 if (strcmp(type->name, t->name) == 0) {
1167 pr_info("Tracer %s already registered\n",
1174 if (!type->set_flag)
1175 type->set_flag = &dummy_set_flag;
1177 type->flags = &dummy_tracer_flags;
1179 if (!type->flags->opts)
1180 type->flags->opts = dummy_tracer_opt;
1182 ret = run_tracer_selftest(type);
1186 type->next = trace_types;
1190 tracing_selftest_running = false;
1191 mutex_unlock(&trace_types_lock);
1193 if (ret || !default_bootup_tracer)
1196 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1199 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1200 /* Do we want this tracer to start on bootup? */
1201 tracing_set_tracer(&global_trace, type->name);
1202 default_bootup_tracer = NULL;
1203 /* disable other selftests, since this will break it. */
1204 tracing_selftest_disabled = true;
1205 #ifdef CONFIG_FTRACE_STARTUP_TEST
1206 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1214 void tracing_reset(struct trace_buffer *buf, int cpu)
1216 struct ring_buffer *buffer = buf->buffer;
1221 ring_buffer_record_disable(buffer);
1223 /* Make sure all commits have finished */
1224 synchronize_sched();
1225 ring_buffer_reset_cpu(buffer, cpu);
1227 ring_buffer_record_enable(buffer);
1230 void tracing_reset_online_cpus(struct trace_buffer *buf)
1232 struct ring_buffer *buffer = buf->buffer;
1238 ring_buffer_record_disable(buffer);
1240 /* Make sure all commits have finished */
1241 synchronize_sched();
1243 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1245 for_each_online_cpu(cpu)
1246 ring_buffer_reset_cpu(buffer, cpu);
1248 ring_buffer_record_enable(buffer);
1251 /* Must have trace_types_lock held */
1252 void tracing_reset_all_online_cpus(void)
1254 struct trace_array *tr;
1256 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1257 tracing_reset_online_cpus(&tr->trace_buffer);
1258 #ifdef CONFIG_TRACER_MAX_TRACE
1259 tracing_reset_online_cpus(&tr->max_buffer);
1264 #define SAVED_CMDLINES_DEFAULT 128
1265 #define NO_CMDLINE_MAP UINT_MAX
1266 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1267 struct saved_cmdlines_buffer {
1268 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1269 unsigned *map_cmdline_to_pid;
1270 unsigned cmdline_num;
1272 char *saved_cmdlines;
1274 static struct saved_cmdlines_buffer *savedcmd;
1276 /* temporary disable recording */
1277 static atomic_t trace_record_cmdline_disabled __read_mostly;
1279 static inline char *get_saved_cmdlines(int idx)
1281 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1284 static inline void set_cmdline(int idx, const char *cmdline)
1286 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1289 static int allocate_cmdlines_buffer(unsigned int val,
1290 struct saved_cmdlines_buffer *s)
1292 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1294 if (!s->map_cmdline_to_pid)
1297 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1298 if (!s->saved_cmdlines) {
1299 kfree(s->map_cmdline_to_pid);
1304 s->cmdline_num = val;
1305 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1306 sizeof(s->map_pid_to_cmdline));
1307 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1308 val * sizeof(*s->map_cmdline_to_pid));
1313 static int trace_create_savedcmd(void)
1317 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1321 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1331 int is_tracing_stopped(void)
1333 return global_trace.stop_count;
1337 * tracing_start - quick start of the tracer
1339 * If tracing is enabled but was stopped by tracing_stop,
1340 * this will start the tracer back up.
1342 void tracing_start(void)
1344 struct ring_buffer *buffer;
1345 unsigned long flags;
1347 if (tracing_disabled)
1350 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1351 if (--global_trace.stop_count) {
1352 if (global_trace.stop_count < 0) {
1353 /* Someone screwed up their debugging */
1355 global_trace.stop_count = 0;
1360 /* Prevent the buffers from switching */
1361 arch_spin_lock(&global_trace.max_lock);
1363 buffer = global_trace.trace_buffer.buffer;
1365 ring_buffer_record_enable(buffer);
1367 #ifdef CONFIG_TRACER_MAX_TRACE
1368 buffer = global_trace.max_buffer.buffer;
1370 ring_buffer_record_enable(buffer);
1373 arch_spin_unlock(&global_trace.max_lock);
1376 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1379 static void tracing_start_tr(struct trace_array *tr)
1381 struct ring_buffer *buffer;
1382 unsigned long flags;
1384 if (tracing_disabled)
1387 /* If global, we need to also start the max tracer */
1388 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1389 return tracing_start();
1391 raw_spin_lock_irqsave(&tr->start_lock, flags);
1393 if (--tr->stop_count) {
1394 if (tr->stop_count < 0) {
1395 /* Someone screwed up their debugging */
1402 buffer = tr->trace_buffer.buffer;
1404 ring_buffer_record_enable(buffer);
1407 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1411 * tracing_stop - quick stop of the tracer
1413 * Light weight way to stop tracing. Use in conjunction with
1416 void tracing_stop(void)
1418 struct ring_buffer *buffer;
1419 unsigned long flags;
1421 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1422 if (global_trace.stop_count++)
1425 /* Prevent the buffers from switching */
1426 arch_spin_lock(&global_trace.max_lock);
1428 buffer = global_trace.trace_buffer.buffer;
1430 ring_buffer_record_disable(buffer);
1432 #ifdef CONFIG_TRACER_MAX_TRACE
1433 buffer = global_trace.max_buffer.buffer;
1435 ring_buffer_record_disable(buffer);
1438 arch_spin_unlock(&global_trace.max_lock);
1441 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1444 static void tracing_stop_tr(struct trace_array *tr)
1446 struct ring_buffer *buffer;
1447 unsigned long flags;
1449 /* If global, we need to also stop the max tracer */
1450 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1451 return tracing_stop();
1453 raw_spin_lock_irqsave(&tr->start_lock, flags);
1454 if (tr->stop_count++)
1457 buffer = tr->trace_buffer.buffer;
1459 ring_buffer_record_disable(buffer);
1462 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1465 void trace_stop_cmdline_recording(void);
1467 static int trace_save_cmdline(struct task_struct *tsk)
1471 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1475 * It's not the end of the world if we don't get
1476 * the lock, but we also don't want to spin
1477 * nor do we want to disable interrupts,
1478 * so if we miss here, then better luck next time.
1480 if (!arch_spin_trylock(&trace_cmdline_lock))
1483 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1484 if (idx == NO_CMDLINE_MAP) {
1485 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1488 * Check whether the cmdline buffer at idx has a pid
1489 * mapped. We are going to overwrite that entry so we
1490 * need to clear the map_pid_to_cmdline. Otherwise we
1491 * would read the new comm for the old pid.
1493 pid = savedcmd->map_cmdline_to_pid[idx];
1494 if (pid != NO_CMDLINE_MAP)
1495 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1497 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1498 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1500 savedcmd->cmdline_idx = idx;
1503 set_cmdline(idx, tsk->comm);
1505 arch_spin_unlock(&trace_cmdline_lock);
1510 static void __trace_find_cmdline(int pid, char comm[])
1515 strcpy(comm, "<idle>");
1519 if (WARN_ON_ONCE(pid < 0)) {
1520 strcpy(comm, "<XXX>");
1524 if (pid > PID_MAX_DEFAULT) {
1525 strcpy(comm, "<...>");
1529 map = savedcmd->map_pid_to_cmdline[pid];
1530 if (map != NO_CMDLINE_MAP)
1531 strcpy(comm, get_saved_cmdlines(map));
1533 strcpy(comm, "<...>");
1536 void trace_find_cmdline(int pid, char comm[])
1539 arch_spin_lock(&trace_cmdline_lock);
1541 __trace_find_cmdline(pid, comm);
1543 arch_spin_unlock(&trace_cmdline_lock);
1547 void tracing_record_cmdline(struct task_struct *tsk)
1549 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1552 if (!__this_cpu_read(trace_cmdline_save))
1555 if (trace_save_cmdline(tsk))
1556 __this_cpu_write(trace_cmdline_save, false);
1560 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1563 struct task_struct *tsk = current;
1565 entry->preempt_count = pc & 0xff;
1566 entry->pid = (tsk) ? tsk->pid : 0;
1568 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1569 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1571 TRACE_FLAG_IRQS_NOSUPPORT |
1573 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1574 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1575 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1576 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1578 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1580 struct ring_buffer_event *
1581 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1584 unsigned long flags, int pc)
1586 struct ring_buffer_event *event;
1588 event = ring_buffer_lock_reserve(buffer, len);
1589 if (event != NULL) {
1590 struct trace_entry *ent = ring_buffer_event_data(event);
1592 tracing_generic_entry_update(ent, flags, pc);
1600 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1602 __this_cpu_write(trace_cmdline_save, true);
1603 ring_buffer_unlock_commit(buffer, event);
1607 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1608 struct ring_buffer_event *event,
1609 unsigned long flags, int pc)
1611 __buffer_unlock_commit(buffer, event);
1613 ftrace_trace_stack(buffer, flags, 6, pc);
1614 ftrace_trace_userstack(buffer, flags, pc);
1617 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1618 struct ring_buffer_event *event,
1619 unsigned long flags, int pc)
1621 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1623 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1625 static struct ring_buffer *temp_buffer;
1627 struct ring_buffer_event *
1628 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1629 struct ftrace_event_file *ftrace_file,
1630 int type, unsigned long len,
1631 unsigned long flags, int pc)
1633 struct ring_buffer_event *entry;
1635 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1636 entry = trace_buffer_lock_reserve(*current_rb,
1637 type, len, flags, pc);
1639 * If tracing is off, but we have triggers enabled
1640 * we still need to look at the event data. Use the temp_buffer
1641 * to store the trace event for the tigger to use. It's recusive
1642 * safe and will not be recorded anywhere.
1644 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1645 *current_rb = temp_buffer;
1646 entry = trace_buffer_lock_reserve(*current_rb,
1647 type, len, flags, pc);
1651 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1653 struct ring_buffer_event *
1654 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1655 int type, unsigned long len,
1656 unsigned long flags, int pc)
1658 *current_rb = global_trace.trace_buffer.buffer;
1659 return trace_buffer_lock_reserve(*current_rb,
1660 type, len, flags, pc);
1662 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1664 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1665 struct ring_buffer_event *event,
1666 unsigned long flags, int pc)
1668 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1670 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1672 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1673 struct ring_buffer_event *event,
1674 unsigned long flags, int pc,
1675 struct pt_regs *regs)
1677 __buffer_unlock_commit(buffer, event);
1679 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1680 ftrace_trace_userstack(buffer, flags, pc);
1682 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1684 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1685 struct ring_buffer_event *event)
1687 ring_buffer_discard_commit(buffer, event);
1689 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1692 trace_function(struct trace_array *tr,
1693 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1696 struct ftrace_event_call *call = &event_function;
1697 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1698 struct ring_buffer_event *event;
1699 struct ftrace_entry *entry;
1701 /* If we are reading the ring buffer, don't trace */
1702 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1705 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1709 entry = ring_buffer_event_data(event);
1711 entry->parent_ip = parent_ip;
1713 if (!call_filter_check_discard(call, entry, buffer, event))
1714 __buffer_unlock_commit(buffer, event);
1717 #ifdef CONFIG_STACKTRACE
1719 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1720 struct ftrace_stack {
1721 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1724 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1725 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1727 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1728 unsigned long flags,
1729 int skip, int pc, struct pt_regs *regs)
1731 struct ftrace_event_call *call = &event_kernel_stack;
1732 struct ring_buffer_event *event;
1733 struct stack_entry *entry;
1734 struct stack_trace trace;
1736 int size = FTRACE_STACK_ENTRIES;
1738 trace.nr_entries = 0;
1742 * Since events can happen in NMIs there's no safe way to
1743 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1744 * or NMI comes in, it will just have to use the default
1745 * FTRACE_STACK_SIZE.
1747 preempt_disable_notrace();
1749 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1751 * We don't need any atomic variables, just a barrier.
1752 * If an interrupt comes in, we don't care, because it would
1753 * have exited and put the counter back to what we want.
1754 * We just need a barrier to keep gcc from moving things
1758 if (use_stack == 1) {
1759 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1760 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1763 save_stack_trace_regs(regs, &trace);
1765 save_stack_trace(&trace);
1767 if (trace.nr_entries > size)
1768 size = trace.nr_entries;
1770 /* From now on, use_stack is a boolean */
1773 size *= sizeof(unsigned long);
1775 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1776 sizeof(*entry) + size, flags, pc);
1779 entry = ring_buffer_event_data(event);
1781 memset(&entry->caller, 0, size);
1784 memcpy(&entry->caller, trace.entries,
1785 trace.nr_entries * sizeof(unsigned long));
1787 trace.max_entries = FTRACE_STACK_ENTRIES;
1788 trace.entries = entry->caller;
1790 save_stack_trace_regs(regs, &trace);
1792 save_stack_trace(&trace);
1795 entry->size = trace.nr_entries;
1797 if (!call_filter_check_discard(call, entry, buffer, event))
1798 __buffer_unlock_commit(buffer, event);
1801 /* Again, don't let gcc optimize things here */
1803 __this_cpu_dec(ftrace_stack_reserve);
1804 preempt_enable_notrace();
1808 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1809 int skip, int pc, struct pt_regs *regs)
1811 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1814 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1817 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1820 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1823 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1826 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1829 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1833 * trace_dump_stack - record a stack back trace in the trace buffer
1834 * @skip: Number of functions to skip (helper handlers)
1836 void trace_dump_stack(int skip)
1838 unsigned long flags;
1840 if (tracing_disabled || tracing_selftest_running)
1843 local_save_flags(flags);
1846 * Skip 3 more, seems to get us at the caller of
1850 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1851 flags, skip, preempt_count(), NULL);
1854 static DEFINE_PER_CPU(int, user_stack_count);
1857 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1859 struct ftrace_event_call *call = &event_user_stack;
1860 struct ring_buffer_event *event;
1861 struct userstack_entry *entry;
1862 struct stack_trace trace;
1864 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1868 * NMIs can not handle page faults, even with fix ups.
1869 * The save user stack can (and often does) fault.
1871 if (unlikely(in_nmi()))
1875 * prevent recursion, since the user stack tracing may
1876 * trigger other kernel events.
1879 if (__this_cpu_read(user_stack_count))
1882 __this_cpu_inc(user_stack_count);
1884 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1885 sizeof(*entry), flags, pc);
1887 goto out_drop_count;
1888 entry = ring_buffer_event_data(event);
1890 entry->tgid = current->tgid;
1891 memset(&entry->caller, 0, sizeof(entry->caller));
1893 trace.nr_entries = 0;
1894 trace.max_entries = FTRACE_STACK_ENTRIES;
1896 trace.entries = entry->caller;
1898 save_stack_trace_user(&trace);
1899 if (!call_filter_check_discard(call, entry, buffer, event))
1900 __buffer_unlock_commit(buffer, event);
1903 __this_cpu_dec(user_stack_count);
1909 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1911 ftrace_trace_userstack(tr, flags, preempt_count());
1915 #endif /* CONFIG_STACKTRACE */
1917 /* created for use with alloc_percpu */
1918 struct trace_buffer_struct {
1919 char buffer[TRACE_BUF_SIZE];
1922 static struct trace_buffer_struct *trace_percpu_buffer;
1923 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1924 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1925 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1928 * The buffer used is dependent on the context. There is a per cpu
1929 * buffer for normal context, softirq contex, hard irq context and
1930 * for NMI context. Thise allows for lockless recording.
1932 * Note, if the buffers failed to be allocated, then this returns NULL
1934 static char *get_trace_buf(void)
1936 struct trace_buffer_struct *percpu_buffer;
1939 * If we have allocated per cpu buffers, then we do not
1940 * need to do any locking.
1943 percpu_buffer = trace_percpu_nmi_buffer;
1945 percpu_buffer = trace_percpu_irq_buffer;
1946 else if (in_softirq())
1947 percpu_buffer = trace_percpu_sirq_buffer;
1949 percpu_buffer = trace_percpu_buffer;
1954 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1957 static int alloc_percpu_trace_buffer(void)
1959 struct trace_buffer_struct *buffers;
1960 struct trace_buffer_struct *sirq_buffers;
1961 struct trace_buffer_struct *irq_buffers;
1962 struct trace_buffer_struct *nmi_buffers;
1964 buffers = alloc_percpu(struct trace_buffer_struct);
1968 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1972 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1976 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1980 trace_percpu_buffer = buffers;
1981 trace_percpu_sirq_buffer = sirq_buffers;
1982 trace_percpu_irq_buffer = irq_buffers;
1983 trace_percpu_nmi_buffer = nmi_buffers;
1988 free_percpu(irq_buffers);
1990 free_percpu(sirq_buffers);
1992 free_percpu(buffers);
1994 WARN(1, "Could not allocate percpu trace_printk buffer");
1998 static int buffers_allocated;
2000 void trace_printk_init_buffers(void)
2002 if (buffers_allocated)
2005 if (alloc_percpu_trace_buffer())
2008 /* trace_printk() is for debug use only. Don't use it in production. */
2010 pr_warning("\n**********************************************************\n");
2011 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2012 pr_warning("** **\n");
2013 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2014 pr_warning("** **\n");
2015 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2016 pr_warning("** unsafe for produciton use. **\n");
2017 pr_warning("** **\n");
2018 pr_warning("** If you see this message and you are not debugging **\n");
2019 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2020 pr_warning("** **\n");
2021 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2022 pr_warning("**********************************************************\n");
2024 /* Expand the buffers to set size */
2025 tracing_update_buffers();
2027 buffers_allocated = 1;
2030 * trace_printk_init_buffers() can be called by modules.
2031 * If that happens, then we need to start cmdline recording
2032 * directly here. If the global_trace.buffer is already
2033 * allocated here, then this was called by module code.
2035 if (global_trace.trace_buffer.buffer)
2036 tracing_start_cmdline_record();
2039 void trace_printk_start_comm(void)
2041 /* Start tracing comms if trace printk is set */
2042 if (!buffers_allocated)
2044 tracing_start_cmdline_record();
2047 static void trace_printk_start_stop_comm(int enabled)
2049 if (!buffers_allocated)
2053 tracing_start_cmdline_record();
2055 tracing_stop_cmdline_record();
2059 * trace_vbprintk - write binary msg to tracing buffer
2062 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2064 struct ftrace_event_call *call = &event_bprint;
2065 struct ring_buffer_event *event;
2066 struct ring_buffer *buffer;
2067 struct trace_array *tr = &global_trace;
2068 struct bprint_entry *entry;
2069 unsigned long flags;
2071 int len = 0, size, pc;
2073 if (unlikely(tracing_selftest_running || tracing_disabled))
2076 /* Don't pollute graph traces with trace_vprintk internals */
2077 pause_graph_tracing();
2079 pc = preempt_count();
2080 preempt_disable_notrace();
2082 tbuffer = get_trace_buf();
2088 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2090 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2093 local_save_flags(flags);
2094 size = sizeof(*entry) + sizeof(u32) * len;
2095 buffer = tr->trace_buffer.buffer;
2096 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2100 entry = ring_buffer_event_data(event);
2104 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2105 if (!call_filter_check_discard(call, entry, buffer, event)) {
2106 __buffer_unlock_commit(buffer, event);
2107 ftrace_trace_stack(buffer, flags, 6, pc);
2111 preempt_enable_notrace();
2112 unpause_graph_tracing();
2116 EXPORT_SYMBOL_GPL(trace_vbprintk);
2119 __trace_array_vprintk(struct ring_buffer *buffer,
2120 unsigned long ip, const char *fmt, va_list args)
2122 struct ftrace_event_call *call = &event_print;
2123 struct ring_buffer_event *event;
2124 int len = 0, size, pc;
2125 struct print_entry *entry;
2126 unsigned long flags;
2129 if (tracing_disabled || tracing_selftest_running)
2132 /* Don't pollute graph traces with trace_vprintk internals */
2133 pause_graph_tracing();
2135 pc = preempt_count();
2136 preempt_disable_notrace();
2139 tbuffer = get_trace_buf();
2145 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2146 if (len > TRACE_BUF_SIZE)
2149 local_save_flags(flags);
2150 size = sizeof(*entry) + len + 1;
2151 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2155 entry = ring_buffer_event_data(event);
2158 memcpy(&entry->buf, tbuffer, len);
2159 entry->buf[len] = '\0';
2160 if (!call_filter_check_discard(call, entry, buffer, event)) {
2161 __buffer_unlock_commit(buffer, event);
2162 ftrace_trace_stack(buffer, flags, 6, pc);
2165 preempt_enable_notrace();
2166 unpause_graph_tracing();
2171 int trace_array_vprintk(struct trace_array *tr,
2172 unsigned long ip, const char *fmt, va_list args)
2174 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2177 int trace_array_printk(struct trace_array *tr,
2178 unsigned long ip, const char *fmt, ...)
2183 if (!(trace_flags & TRACE_ITER_PRINTK))
2187 ret = trace_array_vprintk(tr, ip, fmt, ap);
2192 int trace_array_printk_buf(struct ring_buffer *buffer,
2193 unsigned long ip, const char *fmt, ...)
2198 if (!(trace_flags & TRACE_ITER_PRINTK))
2202 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2207 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2209 return trace_array_vprintk(&global_trace, ip, fmt, args);
2211 EXPORT_SYMBOL_GPL(trace_vprintk);
2213 static void trace_iterator_increment(struct trace_iterator *iter)
2215 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2219 ring_buffer_read(buf_iter, NULL);
2222 static struct trace_entry *
2223 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2224 unsigned long *lost_events)
2226 struct ring_buffer_event *event;
2227 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2230 event = ring_buffer_iter_peek(buf_iter, ts);
2232 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2236 iter->ent_size = ring_buffer_event_length(event);
2237 return ring_buffer_event_data(event);
2243 static struct trace_entry *
2244 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2245 unsigned long *missing_events, u64 *ent_ts)
2247 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2248 struct trace_entry *ent, *next = NULL;
2249 unsigned long lost_events = 0, next_lost = 0;
2250 int cpu_file = iter->cpu_file;
2251 u64 next_ts = 0, ts;
2257 * If we are in a per_cpu trace file, don't bother by iterating over
2258 * all cpu and peek directly.
2260 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2261 if (ring_buffer_empty_cpu(buffer, cpu_file))
2263 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2265 *ent_cpu = cpu_file;
2270 for_each_tracing_cpu(cpu) {
2272 if (ring_buffer_empty_cpu(buffer, cpu))
2275 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2278 * Pick the entry with the smallest timestamp:
2280 if (ent && (!next || ts < next_ts)) {
2284 next_lost = lost_events;
2285 next_size = iter->ent_size;
2289 iter->ent_size = next_size;
2292 *ent_cpu = next_cpu;
2298 *missing_events = next_lost;
2303 /* Find the next real entry, without updating the iterator itself */
2304 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2305 int *ent_cpu, u64 *ent_ts)
2307 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2310 /* Find the next real entry, and increment the iterator to the next entry */
2311 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2313 iter->ent = __find_next_entry(iter, &iter->cpu,
2314 &iter->lost_events, &iter->ts);
2317 trace_iterator_increment(iter);
2319 return iter->ent ? iter : NULL;
2322 static void trace_consume(struct trace_iterator *iter)
2324 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2325 &iter->lost_events);
2328 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2330 struct trace_iterator *iter = m->private;
2334 WARN_ON_ONCE(iter->leftover);
2338 /* can't go backwards */
2343 ent = trace_find_next_entry_inc(iter);
2347 while (ent && iter->idx < i)
2348 ent = trace_find_next_entry_inc(iter);
2355 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2357 struct ring_buffer_event *event;
2358 struct ring_buffer_iter *buf_iter;
2359 unsigned long entries = 0;
2362 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2364 buf_iter = trace_buffer_iter(iter, cpu);
2368 ring_buffer_iter_reset(buf_iter);
2371 * We could have the case with the max latency tracers
2372 * that a reset never took place on a cpu. This is evident
2373 * by the timestamp being before the start of the buffer.
2375 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2376 if (ts >= iter->trace_buffer->time_start)
2379 ring_buffer_read(buf_iter, NULL);
2382 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2386 * The current tracer is copied to avoid a global locking
2389 static void *s_start(struct seq_file *m, loff_t *pos)
2391 struct trace_iterator *iter = m->private;
2392 struct trace_array *tr = iter->tr;
2393 int cpu_file = iter->cpu_file;
2399 * copy the tracer to avoid using a global lock all around.
2400 * iter->trace is a copy of current_trace, the pointer to the
2401 * name may be used instead of a strcmp(), as iter->trace->name
2402 * will point to the same string as current_trace->name.
2404 mutex_lock(&trace_types_lock);
2405 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2406 *iter->trace = *tr->current_trace;
2407 mutex_unlock(&trace_types_lock);
2409 #ifdef CONFIG_TRACER_MAX_TRACE
2410 if (iter->snapshot && iter->trace->use_max_tr)
2411 return ERR_PTR(-EBUSY);
2414 if (!iter->snapshot)
2415 atomic_inc(&trace_record_cmdline_disabled);
2417 if (*pos != iter->pos) {
2422 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2423 for_each_tracing_cpu(cpu)
2424 tracing_iter_reset(iter, cpu);
2426 tracing_iter_reset(iter, cpu_file);
2429 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2434 * If we overflowed the seq_file before, then we want
2435 * to just reuse the trace_seq buffer again.
2441 p = s_next(m, p, &l);
2445 trace_event_read_lock();
2446 trace_access_lock(cpu_file);
2450 static void s_stop(struct seq_file *m, void *p)
2452 struct trace_iterator *iter = m->private;
2454 #ifdef CONFIG_TRACER_MAX_TRACE
2455 if (iter->snapshot && iter->trace->use_max_tr)
2459 if (!iter->snapshot)
2460 atomic_dec(&trace_record_cmdline_disabled);
2462 trace_access_unlock(iter->cpu_file);
2463 trace_event_read_unlock();
2467 get_total_entries(struct trace_buffer *buf,
2468 unsigned long *total, unsigned long *entries)
2470 unsigned long count;
2476 for_each_tracing_cpu(cpu) {
2477 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2479 * If this buffer has skipped entries, then we hold all
2480 * entries for the trace and we need to ignore the
2481 * ones before the time stamp.
2483 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2484 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2485 /* total is the same as the entries */
2489 ring_buffer_overrun_cpu(buf->buffer, cpu);
2494 static void print_lat_help_header(struct seq_file *m)
2496 seq_puts(m, "# _------=> CPU# \n");
2497 seq_puts(m, "# / _-----=> irqs-off \n");
2498 seq_puts(m, "# | / _----=> need-resched \n");
2499 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2500 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2501 seq_puts(m, "# |||| / delay \n");
2502 seq_puts(m, "# cmd pid ||||| time | caller \n");
2503 seq_puts(m, "# \\ / ||||| \\ | / \n");
2506 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2508 unsigned long total;
2509 unsigned long entries;
2511 get_total_entries(buf, &total, &entries);
2512 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2513 entries, total, num_online_cpus());
2517 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2519 print_event_info(buf, m);
2520 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2521 seq_puts(m, "# | | | | |\n");
2524 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2526 print_event_info(buf, m);
2527 seq_puts(m, "# _-----=> irqs-off\n");
2528 seq_puts(m, "# / _----=> need-resched\n");
2529 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2530 seq_puts(m, "# || / _--=> preempt-depth\n");
2531 seq_puts(m, "# ||| / delay\n");
2532 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2533 seq_puts(m, "# | | | |||| | |\n");
2537 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2539 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2540 struct trace_buffer *buf = iter->trace_buffer;
2541 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2542 struct tracer *type = iter->trace;
2543 unsigned long entries;
2544 unsigned long total;
2545 const char *name = "preemption";
2549 get_total_entries(buf, &total, &entries);
2551 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2553 seq_puts(m, "# -----------------------------------"
2554 "---------------------------------\n");
2555 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2556 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2557 nsecs_to_usecs(data->saved_latency),
2561 #if defined(CONFIG_PREEMPT_NONE)
2563 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2565 #elif defined(CONFIG_PREEMPT)
2570 /* These are reserved for later use */
2573 seq_printf(m, " #P:%d)\n", num_online_cpus());
2577 seq_puts(m, "# -----------------\n");
2578 seq_printf(m, "# | task: %.16s-%d "
2579 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2580 data->comm, data->pid,
2581 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2582 data->policy, data->rt_priority);
2583 seq_puts(m, "# -----------------\n");
2585 if (data->critical_start) {
2586 seq_puts(m, "# => started at: ");
2587 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2588 trace_print_seq(m, &iter->seq);
2589 seq_puts(m, "\n# => ended at: ");
2590 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2591 trace_print_seq(m, &iter->seq);
2592 seq_puts(m, "\n#\n");
2598 static void test_cpu_buff_start(struct trace_iterator *iter)
2600 struct trace_seq *s = &iter->seq;
2602 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2605 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2608 if (cpumask_test_cpu(iter->cpu, iter->started))
2611 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2614 cpumask_set_cpu(iter->cpu, iter->started);
2616 /* Don't print started cpu buffer for the first entry of the trace */
2618 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2622 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2624 struct trace_seq *s = &iter->seq;
2625 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2626 struct trace_entry *entry;
2627 struct trace_event *event;
2631 test_cpu_buff_start(iter);
2633 event = ftrace_find_event(entry->type);
2635 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2636 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2637 if (!trace_print_lat_context(iter))
2640 if (!trace_print_context(iter))
2646 return event->funcs->trace(iter, sym_flags, event);
2648 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2651 return TRACE_TYPE_HANDLED;
2653 return TRACE_TYPE_PARTIAL_LINE;
2656 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2658 struct trace_seq *s = &iter->seq;
2659 struct trace_entry *entry;
2660 struct trace_event *event;
2664 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2665 if (!trace_seq_printf(s, "%d %d %llu ",
2666 entry->pid, iter->cpu, iter->ts))
2670 event = ftrace_find_event(entry->type);
2672 return event->funcs->raw(iter, 0, event);
2674 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2677 return TRACE_TYPE_HANDLED;
2679 return TRACE_TYPE_PARTIAL_LINE;
2682 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2684 struct trace_seq *s = &iter->seq;
2685 unsigned char newline = '\n';
2686 struct trace_entry *entry;
2687 struct trace_event *event;
2691 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2692 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2693 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2694 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2697 event = ftrace_find_event(entry->type);
2699 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2700 if (ret != TRACE_TYPE_HANDLED)
2704 SEQ_PUT_FIELD_RET(s, newline);
2706 return TRACE_TYPE_HANDLED;
2709 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2711 struct trace_seq *s = &iter->seq;
2712 struct trace_entry *entry;
2713 struct trace_event *event;
2717 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2718 SEQ_PUT_FIELD_RET(s, entry->pid);
2719 SEQ_PUT_FIELD_RET(s, iter->cpu);
2720 SEQ_PUT_FIELD_RET(s, iter->ts);
2723 event = ftrace_find_event(entry->type);
2724 return event ? event->funcs->binary(iter, 0, event) :
2728 int trace_empty(struct trace_iterator *iter)
2730 struct ring_buffer_iter *buf_iter;
2733 /* If we are looking at one CPU buffer, only check that one */
2734 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2735 cpu = iter->cpu_file;
2736 buf_iter = trace_buffer_iter(iter, cpu);
2738 if (!ring_buffer_iter_empty(buf_iter))
2741 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2747 for_each_tracing_cpu(cpu) {
2748 buf_iter = trace_buffer_iter(iter, cpu);
2750 if (!ring_buffer_iter_empty(buf_iter))
2753 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2761 /* Called with trace_event_read_lock() held. */
2762 enum print_line_t print_trace_line(struct trace_iterator *iter)
2764 enum print_line_t ret;
2766 if (iter->lost_events &&
2767 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2768 iter->cpu, iter->lost_events))
2769 return TRACE_TYPE_PARTIAL_LINE;
2771 if (iter->trace && iter->trace->print_line) {
2772 ret = iter->trace->print_line(iter);
2773 if (ret != TRACE_TYPE_UNHANDLED)
2777 if (iter->ent->type == TRACE_BPUTS &&
2778 trace_flags & TRACE_ITER_PRINTK &&
2779 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2780 return trace_print_bputs_msg_only(iter);
2782 if (iter->ent->type == TRACE_BPRINT &&
2783 trace_flags & TRACE_ITER_PRINTK &&
2784 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2785 return trace_print_bprintk_msg_only(iter);
2787 if (iter->ent->type == TRACE_PRINT &&
2788 trace_flags & TRACE_ITER_PRINTK &&
2789 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2790 return trace_print_printk_msg_only(iter);
2792 if (trace_flags & TRACE_ITER_BIN)
2793 return print_bin_fmt(iter);
2795 if (trace_flags & TRACE_ITER_HEX)
2796 return print_hex_fmt(iter);
2798 if (trace_flags & TRACE_ITER_RAW)
2799 return print_raw_fmt(iter);
2801 return print_trace_fmt(iter);
2804 void trace_latency_header(struct seq_file *m)
2806 struct trace_iterator *iter = m->private;
2808 /* print nothing if the buffers are empty */
2809 if (trace_empty(iter))
2812 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2813 print_trace_header(m, iter);
2815 if (!(trace_flags & TRACE_ITER_VERBOSE))
2816 print_lat_help_header(m);
2819 void trace_default_header(struct seq_file *m)
2821 struct trace_iterator *iter = m->private;
2823 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2826 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2827 /* print nothing if the buffers are empty */
2828 if (trace_empty(iter))
2830 print_trace_header(m, iter);
2831 if (!(trace_flags & TRACE_ITER_VERBOSE))
2832 print_lat_help_header(m);
2834 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2835 if (trace_flags & TRACE_ITER_IRQ_INFO)
2836 print_func_help_header_irq(iter->trace_buffer, m);
2838 print_func_help_header(iter->trace_buffer, m);
2843 static void test_ftrace_alive(struct seq_file *m)
2845 if (!ftrace_is_dead())
2847 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2848 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2851 #ifdef CONFIG_TRACER_MAX_TRACE
2852 static void show_snapshot_main_help(struct seq_file *m)
2854 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2855 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2856 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2857 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2858 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2859 seq_printf(m, "# is not a '0' or '1')\n");
2862 static void show_snapshot_percpu_help(struct seq_file *m)
2864 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2865 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2866 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2867 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2869 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2870 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2872 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2873 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2874 seq_printf(m, "# is not a '0' or '1')\n");
2877 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2879 if (iter->tr->allocated_snapshot)
2880 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2882 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2884 seq_printf(m, "# Snapshot commands:\n");
2885 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2886 show_snapshot_main_help(m);
2888 show_snapshot_percpu_help(m);
2891 /* Should never be called */
2892 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2895 static int s_show(struct seq_file *m, void *v)
2897 struct trace_iterator *iter = v;
2900 if (iter->ent == NULL) {
2902 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2904 test_ftrace_alive(m);
2906 if (iter->snapshot && trace_empty(iter))
2907 print_snapshot_help(m, iter);
2908 else if (iter->trace && iter->trace->print_header)
2909 iter->trace->print_header(m);
2911 trace_default_header(m);
2913 } else if (iter->leftover) {
2915 * If we filled the seq_file buffer earlier, we
2916 * want to just show it now.
2918 ret = trace_print_seq(m, &iter->seq);
2920 /* ret should this time be zero, but you never know */
2921 iter->leftover = ret;
2924 print_trace_line(iter);
2925 ret = trace_print_seq(m, &iter->seq);
2927 * If we overflow the seq_file buffer, then it will
2928 * ask us for this data again at start up.
2930 * ret is 0 if seq_file write succeeded.
2933 iter->leftover = ret;
2940 * Should be used after trace_array_get(), trace_types_lock
2941 * ensures that i_cdev was already initialized.
2943 static inline int tracing_get_cpu(struct inode *inode)
2945 if (inode->i_cdev) /* See trace_create_cpu_file() */
2946 return (long)inode->i_cdev - 1;
2947 return RING_BUFFER_ALL_CPUS;
2950 static const struct seq_operations tracer_seq_ops = {
2957 static struct trace_iterator *
2958 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2960 struct trace_array *tr = inode->i_private;
2961 struct trace_iterator *iter;
2964 if (tracing_disabled)
2965 return ERR_PTR(-ENODEV);
2967 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2969 return ERR_PTR(-ENOMEM);
2971 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2973 if (!iter->buffer_iter)
2977 * We make a copy of the current tracer to avoid concurrent
2978 * changes on it while we are reading.
2980 mutex_lock(&trace_types_lock);
2981 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2985 *iter->trace = *tr->current_trace;
2987 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2992 #ifdef CONFIG_TRACER_MAX_TRACE
2993 /* Currently only the top directory has a snapshot */
2994 if (tr->current_trace->print_max || snapshot)
2995 iter->trace_buffer = &tr->max_buffer;
2998 iter->trace_buffer = &tr->trace_buffer;
2999 iter->snapshot = snapshot;
3001 iter->cpu_file = tracing_get_cpu(inode);
3002 mutex_init(&iter->mutex);
3004 /* Notify the tracer early; before we stop tracing. */
3005 if (iter->trace && iter->trace->open)
3006 iter->trace->open(iter);
3008 /* Annotate start of buffers if we had overruns */
3009 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3010 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3012 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3013 if (trace_clocks[tr->clock_id].in_ns)
3014 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3016 /* stop the trace while dumping if we are not opening "snapshot" */
3017 if (!iter->snapshot)
3018 tracing_stop_tr(tr);
3020 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3021 for_each_tracing_cpu(cpu) {
3022 iter->buffer_iter[cpu] =
3023 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3025 ring_buffer_read_prepare_sync();
3026 for_each_tracing_cpu(cpu) {
3027 ring_buffer_read_start(iter->buffer_iter[cpu]);
3028 tracing_iter_reset(iter, cpu);
3031 cpu = iter->cpu_file;
3032 iter->buffer_iter[cpu] =
3033 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3034 ring_buffer_read_prepare_sync();
3035 ring_buffer_read_start(iter->buffer_iter[cpu]);
3036 tracing_iter_reset(iter, cpu);
3039 mutex_unlock(&trace_types_lock);
3044 mutex_unlock(&trace_types_lock);
3046 kfree(iter->buffer_iter);
3048 seq_release_private(inode, file);
3049 return ERR_PTR(-ENOMEM);
3052 int tracing_open_generic(struct inode *inode, struct file *filp)
3054 if (tracing_disabled)
3057 filp->private_data = inode->i_private;
3061 bool tracing_is_disabled(void)
3063 return (tracing_disabled) ? true: false;
3067 * Open and update trace_array ref count.
3068 * Must have the current trace_array passed to it.
3070 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3072 struct trace_array *tr = inode->i_private;
3074 if (tracing_disabled)
3077 if (trace_array_get(tr) < 0)
3080 filp->private_data = inode->i_private;
3085 static int tracing_release(struct inode *inode, struct file *file)
3087 struct trace_array *tr = inode->i_private;
3088 struct seq_file *m = file->private_data;
3089 struct trace_iterator *iter;
3092 if (!(file->f_mode & FMODE_READ)) {
3093 trace_array_put(tr);
3097 /* Writes do not use seq_file */
3099 mutex_lock(&trace_types_lock);
3101 for_each_tracing_cpu(cpu) {
3102 if (iter->buffer_iter[cpu])
3103 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3106 if (iter->trace && iter->trace->close)
3107 iter->trace->close(iter);
3109 if (!iter->snapshot)
3110 /* reenable tracing if it was previously enabled */
3111 tracing_start_tr(tr);
3113 __trace_array_put(tr);
3115 mutex_unlock(&trace_types_lock);
3117 mutex_destroy(&iter->mutex);
3118 free_cpumask_var(iter->started);
3120 kfree(iter->buffer_iter);
3121 seq_release_private(inode, file);
3126 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3128 struct trace_array *tr = inode->i_private;
3130 trace_array_put(tr);
3134 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3136 struct trace_array *tr = inode->i_private;
3138 trace_array_put(tr);
3140 return single_release(inode, file);
3143 static int tracing_open(struct inode *inode, struct file *file)
3145 struct trace_array *tr = inode->i_private;
3146 struct trace_iterator *iter;
3149 if (trace_array_get(tr) < 0)
3152 /* If this file was open for write, then erase contents */
3153 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3154 int cpu = tracing_get_cpu(inode);
3156 if (cpu == RING_BUFFER_ALL_CPUS)
3157 tracing_reset_online_cpus(&tr->trace_buffer);
3159 tracing_reset(&tr->trace_buffer, cpu);
3162 if (file->f_mode & FMODE_READ) {
3163 iter = __tracing_open(inode, file, false);
3165 ret = PTR_ERR(iter);
3166 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3167 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3171 trace_array_put(tr);
3177 * Some tracers are not suitable for instance buffers.
3178 * A tracer is always available for the global array (toplevel)
3179 * or if it explicitly states that it is.
3182 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3184 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3187 /* Find the next tracer that this trace array may use */
3188 static struct tracer *
3189 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3191 while (t && !trace_ok_for_array(t, tr))
3198 t_next(struct seq_file *m, void *v, loff_t *pos)
3200 struct trace_array *tr = m->private;
3201 struct tracer *t = v;
3206 t = get_tracer_for_array(tr, t->next);
3211 static void *t_start(struct seq_file *m, loff_t *pos)
3213 struct trace_array *tr = m->private;
3217 mutex_lock(&trace_types_lock);
3219 t = get_tracer_for_array(tr, trace_types);
3220 for (; t && l < *pos; t = t_next(m, t, &l))
3226 static void t_stop(struct seq_file *m, void *p)
3228 mutex_unlock(&trace_types_lock);
3231 static int t_show(struct seq_file *m, void *v)
3233 struct tracer *t = v;
3238 seq_printf(m, "%s", t->name);
3247 static const struct seq_operations show_traces_seq_ops = {
3254 static int show_traces_open(struct inode *inode, struct file *file)
3256 struct trace_array *tr = inode->i_private;
3260 if (tracing_disabled)
3263 ret = seq_open(file, &show_traces_seq_ops);
3267 m = file->private_data;
3274 tracing_write_stub(struct file *filp, const char __user *ubuf,
3275 size_t count, loff_t *ppos)
3280 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3284 if (file->f_mode & FMODE_READ)
3285 ret = seq_lseek(file, offset, whence);
3287 file->f_pos = ret = 0;
3292 static const struct file_operations tracing_fops = {
3293 .open = tracing_open,
3295 .write = tracing_write_stub,
3296 .llseek = tracing_lseek,
3297 .release = tracing_release,
3300 static const struct file_operations show_traces_fops = {
3301 .open = show_traces_open,
3303 .release = seq_release,
3304 .llseek = seq_lseek,
3308 * The tracer itself will not take this lock, but still we want
3309 * to provide a consistent cpumask to user-space:
3311 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3314 * Temporary storage for the character representation of the
3315 * CPU bitmask (and one more byte for the newline):
3317 static char mask_str[NR_CPUS + 1];
3320 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3321 size_t count, loff_t *ppos)
3323 struct trace_array *tr = file_inode(filp)->i_private;
3326 mutex_lock(&tracing_cpumask_update_lock);
3328 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3329 if (count - len < 2) {
3333 len += sprintf(mask_str + len, "\n");
3334 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3337 mutex_unlock(&tracing_cpumask_update_lock);
3343 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3344 size_t count, loff_t *ppos)
3346 struct trace_array *tr = file_inode(filp)->i_private;
3347 cpumask_var_t tracing_cpumask_new;
3350 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3353 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3357 mutex_lock(&tracing_cpumask_update_lock);
3359 local_irq_disable();
3360 arch_spin_lock(&tr->max_lock);
3361 for_each_tracing_cpu(cpu) {
3363 * Increase/decrease the disabled counter if we are
3364 * about to flip a bit in the cpumask:
3366 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3367 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3368 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3369 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3371 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3372 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3373 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3374 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3377 arch_spin_unlock(&tr->max_lock);
3380 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3382 mutex_unlock(&tracing_cpumask_update_lock);
3383 free_cpumask_var(tracing_cpumask_new);
3388 free_cpumask_var(tracing_cpumask_new);
3393 static const struct file_operations tracing_cpumask_fops = {
3394 .open = tracing_open_generic_tr,
3395 .read = tracing_cpumask_read,
3396 .write = tracing_cpumask_write,
3397 .release = tracing_release_generic_tr,
3398 .llseek = generic_file_llseek,
3401 static int tracing_trace_options_show(struct seq_file *m, void *v)
3403 struct tracer_opt *trace_opts;
3404 struct trace_array *tr = m->private;
3408 mutex_lock(&trace_types_lock);
3409 tracer_flags = tr->current_trace->flags->val;
3410 trace_opts = tr->current_trace->flags->opts;
3412 for (i = 0; trace_options[i]; i++) {
3413 if (trace_flags & (1 << i))
3414 seq_printf(m, "%s\n", trace_options[i]);
3416 seq_printf(m, "no%s\n", trace_options[i]);
3419 for (i = 0; trace_opts[i].name; i++) {
3420 if (tracer_flags & trace_opts[i].bit)
3421 seq_printf(m, "%s\n", trace_opts[i].name);
3423 seq_printf(m, "no%s\n", trace_opts[i].name);
3425 mutex_unlock(&trace_types_lock);
3430 static int __set_tracer_option(struct trace_array *tr,
3431 struct tracer_flags *tracer_flags,
3432 struct tracer_opt *opts, int neg)
3434 struct tracer *trace = tr->current_trace;
3437 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3442 tracer_flags->val &= ~opts->bit;
3444 tracer_flags->val |= opts->bit;
3448 /* Try to assign a tracer specific option */
3449 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3451 struct tracer *trace = tr->current_trace;
3452 struct tracer_flags *tracer_flags = trace->flags;
3453 struct tracer_opt *opts = NULL;
3456 for (i = 0; tracer_flags->opts[i].name; i++) {
3457 opts = &tracer_flags->opts[i];
3459 if (strcmp(cmp, opts->name) == 0)
3460 return __set_tracer_option(tr, trace->flags, opts, neg);
3466 /* Some tracers require overwrite to stay enabled */
3467 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3469 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3475 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3477 /* do nothing if flag is already set */
3478 if (!!(trace_flags & mask) == !!enabled)
3481 /* Give the tracer a chance to approve the change */
3482 if (tr->current_trace->flag_changed)
3483 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3487 trace_flags |= mask;
3489 trace_flags &= ~mask;
3491 if (mask == TRACE_ITER_RECORD_CMD)
3492 trace_event_enable_cmd_record(enabled);
3494 if (mask == TRACE_ITER_OVERWRITE) {
3495 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3496 #ifdef CONFIG_TRACER_MAX_TRACE
3497 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3501 if (mask == TRACE_ITER_PRINTK)
3502 trace_printk_start_stop_comm(enabled);
3507 static int trace_set_options(struct trace_array *tr, char *option)
3514 cmp = strstrip(option);
3516 if (strncmp(cmp, "no", 2) == 0) {
3521 mutex_lock(&trace_types_lock);
3523 for (i = 0; trace_options[i]; i++) {
3524 if (strcmp(cmp, trace_options[i]) == 0) {
3525 ret = set_tracer_flag(tr, 1 << i, !neg);
3530 /* If no option could be set, test the specific tracer options */
3531 if (!trace_options[i])
3532 ret = set_tracer_option(tr, cmp, neg);
3534 mutex_unlock(&trace_types_lock);
3540 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3541 size_t cnt, loff_t *ppos)
3543 struct seq_file *m = filp->private_data;
3544 struct trace_array *tr = m->private;
3548 if (cnt >= sizeof(buf))
3551 if (copy_from_user(&buf, ubuf, cnt))
3556 ret = trace_set_options(tr, buf);
3565 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3567 struct trace_array *tr = inode->i_private;
3570 if (tracing_disabled)
3573 if (trace_array_get(tr) < 0)
3576 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3578 trace_array_put(tr);
3583 static const struct file_operations tracing_iter_fops = {
3584 .open = tracing_trace_options_open,
3586 .llseek = seq_lseek,
3587 .release = tracing_single_release_tr,
3588 .write = tracing_trace_options_write,
3591 static const char readme_msg[] =
3592 "tracing mini-HOWTO:\n\n"
3593 "# echo 0 > tracing_on : quick way to disable tracing\n"
3594 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3595 " Important files:\n"
3596 " trace\t\t\t- The static contents of the buffer\n"
3597 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3598 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3599 " current_tracer\t- function and latency tracers\n"
3600 " available_tracers\t- list of configured tracers for current_tracer\n"
3601 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3602 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3603 " trace_clock\t\t-change the clock used to order events\n"
3604 " local: Per cpu clock but may not be synced across CPUs\n"
3605 " global: Synced across CPUs but slows tracing down.\n"
3606 " counter: Not a clock, but just an increment\n"
3607 " uptime: Jiffy counter from time of boot\n"
3608 " perf: Same clock that perf events use\n"
3609 #ifdef CONFIG_X86_64
3610 " x86-tsc: TSC cycle counter\n"
3612 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3613 " tracing_cpumask\t- Limit which CPUs to trace\n"
3614 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3615 "\t\t\t Remove sub-buffer with rmdir\n"
3616 " trace_options\t\t- Set format or modify how tracing happens\n"
3617 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3618 "\t\t\t option name\n"
3619 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3620 #ifdef CONFIG_DYNAMIC_FTRACE
3621 "\n available_filter_functions - list of functions that can be filtered on\n"
3622 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3623 "\t\t\t functions\n"
3624 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3625 "\t modules: Can select a group via module\n"
3626 "\t Format: :mod:<module-name>\n"
3627 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3628 "\t triggers: a command to perform when function is hit\n"
3629 "\t Format: <function>:<trigger>[:count]\n"
3630 "\t trigger: traceon, traceoff\n"
3631 "\t\t enable_event:<system>:<event>\n"
3632 "\t\t disable_event:<system>:<event>\n"
3633 #ifdef CONFIG_STACKTRACE
3636 #ifdef CONFIG_TRACER_SNAPSHOT
3641 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3642 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3643 "\t The first one will disable tracing every time do_fault is hit\n"
3644 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3645 "\t The first time do trap is hit and it disables tracing, the\n"
3646 "\t counter will decrement to 2. If tracing is already disabled,\n"
3647 "\t the counter will not decrement. It only decrements when the\n"
3648 "\t trigger did work\n"
3649 "\t To remove trigger without count:\n"
3650 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3651 "\t To remove trigger with a count:\n"
3652 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3653 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3654 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3655 "\t modules: Can select a group via module command :mod:\n"
3656 "\t Does not accept triggers\n"
3657 #endif /* CONFIG_DYNAMIC_FTRACE */
3658 #ifdef CONFIG_FUNCTION_TRACER
3659 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3662 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3663 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3664 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3665 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3667 #ifdef CONFIG_TRACER_SNAPSHOT
3668 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3669 "\t\t\t snapshot buffer. Read the contents for more\n"
3670 "\t\t\t information\n"
3672 #ifdef CONFIG_STACK_TRACER
3673 " stack_trace\t\t- Shows the max stack trace when active\n"
3674 " stack_max_size\t- Shows current max stack size that was traced\n"
3675 "\t\t\t Write into this file to reset the max size (trigger a\n"
3676 "\t\t\t new trace)\n"
3677 #ifdef CONFIG_DYNAMIC_FTRACE
3678 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3681 #endif /* CONFIG_STACK_TRACER */
3682 " events/\t\t- Directory containing all trace event subsystems:\n"
3683 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3684 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3685 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3687 " filter\t\t- If set, only events passing filter are traced\n"
3688 " events/<system>/<event>/\t- Directory containing control files for\n"
3690 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3691 " filter\t\t- If set, only events passing filter are traced\n"
3692 " trigger\t\t- If set, a command to perform when event is hit\n"
3693 "\t Format: <trigger>[:count][if <filter>]\n"
3694 "\t trigger: traceon, traceoff\n"
3695 "\t enable_event:<system>:<event>\n"
3696 "\t disable_event:<system>:<event>\n"
3697 #ifdef CONFIG_STACKTRACE
3700 #ifdef CONFIG_TRACER_SNAPSHOT
3703 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3704 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3705 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3706 "\t events/block/block_unplug/trigger\n"
3707 "\t The first disables tracing every time block_unplug is hit.\n"
3708 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3709 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3710 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3711 "\t Like function triggers, the counter is only decremented if it\n"
3712 "\t enabled or disabled tracing.\n"
3713 "\t To remove a trigger without a count:\n"
3714 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3715 "\t To remove a trigger with a count:\n"
3716 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3717 "\t Filters can be ignored when removing a trigger.\n"
3721 tracing_readme_read(struct file *filp, char __user *ubuf,
3722 size_t cnt, loff_t *ppos)
3724 return simple_read_from_buffer(ubuf, cnt, ppos,
3725 readme_msg, strlen(readme_msg));
3728 static const struct file_operations tracing_readme_fops = {
3729 .open = tracing_open_generic,
3730 .read = tracing_readme_read,
3731 .llseek = generic_file_llseek,
3734 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3736 unsigned int *ptr = v;
3738 if (*pos || m->count)
3743 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3745 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3754 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3760 arch_spin_lock(&trace_cmdline_lock);
3762 v = &savedcmd->map_cmdline_to_pid[0];
3764 v = saved_cmdlines_next(m, v, &l);
3772 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3774 arch_spin_unlock(&trace_cmdline_lock);
3778 static int saved_cmdlines_show(struct seq_file *m, void *v)
3780 char buf[TASK_COMM_LEN];
3781 unsigned int *pid = v;
3783 __trace_find_cmdline(*pid, buf);
3784 seq_printf(m, "%d %s\n", *pid, buf);
3788 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3789 .start = saved_cmdlines_start,
3790 .next = saved_cmdlines_next,
3791 .stop = saved_cmdlines_stop,
3792 .show = saved_cmdlines_show,
3795 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3797 if (tracing_disabled)
3800 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3803 static const struct file_operations tracing_saved_cmdlines_fops = {
3804 .open = tracing_saved_cmdlines_open,
3806 .llseek = seq_lseek,
3807 .release = seq_release,
3811 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3812 size_t cnt, loff_t *ppos)
3817 arch_spin_lock(&trace_cmdline_lock);
3818 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3819 arch_spin_unlock(&trace_cmdline_lock);
3821 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3824 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3826 kfree(s->saved_cmdlines);
3827 kfree(s->map_cmdline_to_pid);
3831 static int tracing_resize_saved_cmdlines(unsigned int val)
3833 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3835 s = kmalloc(sizeof(*s), GFP_KERNEL);
3839 if (allocate_cmdlines_buffer(val, s) < 0) {
3844 arch_spin_lock(&trace_cmdline_lock);
3845 savedcmd_temp = savedcmd;
3847 arch_spin_unlock(&trace_cmdline_lock);
3848 free_saved_cmdlines_buffer(savedcmd_temp);
3854 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3855 size_t cnt, loff_t *ppos)
3860 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3864 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3865 if (!val || val > PID_MAX_DEFAULT)
3868 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3877 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3878 .open = tracing_open_generic,
3879 .read = tracing_saved_cmdlines_size_read,
3880 .write = tracing_saved_cmdlines_size_write,
3884 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3885 size_t cnt, loff_t *ppos)
3887 struct trace_array *tr = filp->private_data;
3888 char buf[MAX_TRACER_SIZE+2];
3891 mutex_lock(&trace_types_lock);
3892 r = sprintf(buf, "%s\n", tr->current_trace->name);
3893 mutex_unlock(&trace_types_lock);
3895 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3898 int tracer_init(struct tracer *t, struct trace_array *tr)
3900 tracing_reset_online_cpus(&tr->trace_buffer);
3904 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3908 for_each_tracing_cpu(cpu)
3909 per_cpu_ptr(buf->data, cpu)->entries = val;
3912 #ifdef CONFIG_TRACER_MAX_TRACE
3913 /* resize @tr's buffer to the size of @size_tr's entries */
3914 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3915 struct trace_buffer *size_buf, int cpu_id)
3919 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3920 for_each_tracing_cpu(cpu) {
3921 ret = ring_buffer_resize(trace_buf->buffer,
3922 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3925 per_cpu_ptr(trace_buf->data, cpu)->entries =
3926 per_cpu_ptr(size_buf->data, cpu)->entries;
3929 ret = ring_buffer_resize(trace_buf->buffer,
3930 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3932 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3933 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3938 #endif /* CONFIG_TRACER_MAX_TRACE */
3940 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3941 unsigned long size, int cpu)
3946 * If kernel or user changes the size of the ring buffer
3947 * we use the size that was given, and we can forget about
3948 * expanding it later.
3950 ring_buffer_expanded = true;
3952 /* May be called before buffers are initialized */
3953 if (!tr->trace_buffer.buffer)
3956 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3960 #ifdef CONFIG_TRACER_MAX_TRACE
3961 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3962 !tr->current_trace->use_max_tr)
3965 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3967 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3968 &tr->trace_buffer, cpu);
3971 * AARGH! We are left with different
3972 * size max buffer!!!!
3973 * The max buffer is our "snapshot" buffer.
3974 * When a tracer needs a snapshot (one of the
3975 * latency tracers), it swaps the max buffer
3976 * with the saved snap shot. We succeeded to
3977 * update the size of the main buffer, but failed to
3978 * update the size of the max buffer. But when we tried
3979 * to reset the main buffer to the original size, we
3980 * failed there too. This is very unlikely to
3981 * happen, but if it does, warn and kill all
3985 tracing_disabled = 1;
3990 if (cpu == RING_BUFFER_ALL_CPUS)
3991 set_buffer_entries(&tr->max_buffer, size);
3993 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3996 #endif /* CONFIG_TRACER_MAX_TRACE */
3998 if (cpu == RING_BUFFER_ALL_CPUS)
3999 set_buffer_entries(&tr->trace_buffer, size);
4001 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4006 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4007 unsigned long size, int cpu_id)
4011 mutex_lock(&trace_types_lock);
4013 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4014 /* make sure, this cpu is enabled in the mask */
4015 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4021 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4026 mutex_unlock(&trace_types_lock);
4033 * tracing_update_buffers - used by tracing facility to expand ring buffers
4035 * To save on memory when the tracing is never used on a system with it
4036 * configured in. The ring buffers are set to a minimum size. But once
4037 * a user starts to use the tracing facility, then they need to grow
4038 * to their default size.
4040 * This function is to be called when a tracer is about to be used.
4042 int tracing_update_buffers(void)
4046 mutex_lock(&trace_types_lock);
4047 if (!ring_buffer_expanded)
4048 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4049 RING_BUFFER_ALL_CPUS);
4050 mutex_unlock(&trace_types_lock);
4055 struct trace_option_dentry;
4057 static struct trace_option_dentry *
4058 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4061 destroy_trace_option_files(struct trace_option_dentry *topts);
4064 * Used to clear out the tracer before deletion of an instance.
4065 * Must have trace_types_lock held.
4067 static void tracing_set_nop(struct trace_array *tr)
4069 if (tr->current_trace == &nop_trace)
4072 tr->current_trace->enabled--;
4074 if (tr->current_trace->reset)
4075 tr->current_trace->reset(tr);
4077 tr->current_trace = &nop_trace;
4080 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4082 static struct trace_option_dentry *topts;
4084 #ifdef CONFIG_TRACER_MAX_TRACE
4089 mutex_lock(&trace_types_lock);
4091 if (!ring_buffer_expanded) {
4092 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4093 RING_BUFFER_ALL_CPUS);
4099 for (t = trace_types; t; t = t->next) {
4100 if (strcmp(t->name, buf) == 0)
4107 if (t == tr->current_trace)
4110 /* Some tracers are only allowed for the top level buffer */
4111 if (!trace_ok_for_array(t, tr)) {
4116 trace_branch_disable();
4118 tr->current_trace->enabled--;
4120 if (tr->current_trace->reset)
4121 tr->current_trace->reset(tr);
4123 /* Current trace needs to be nop_trace before synchronize_sched */
4124 tr->current_trace = &nop_trace;
4126 #ifdef CONFIG_TRACER_MAX_TRACE
4127 had_max_tr = tr->allocated_snapshot;
4129 if (had_max_tr && !t->use_max_tr) {
4131 * We need to make sure that the update_max_tr sees that
4132 * current_trace changed to nop_trace to keep it from
4133 * swapping the buffers after we resize it.
4134 * The update_max_tr is called from interrupts disabled
4135 * so a synchronized_sched() is sufficient.
4137 synchronize_sched();
4141 /* Currently, only the top instance has options */
4142 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4143 destroy_trace_option_files(topts);
4144 topts = create_trace_option_files(tr, t);
4147 #ifdef CONFIG_TRACER_MAX_TRACE
4148 if (t->use_max_tr && !had_max_tr) {
4149 ret = alloc_snapshot(tr);
4156 ret = tracer_init(t, tr);
4161 tr->current_trace = t;
4162 tr->current_trace->enabled++;
4163 trace_branch_enable(tr);
4165 mutex_unlock(&trace_types_lock);
4171 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4172 size_t cnt, loff_t *ppos)
4174 struct trace_array *tr = filp->private_data;
4175 char buf[MAX_TRACER_SIZE+1];
4182 if (cnt > MAX_TRACER_SIZE)
4183 cnt = MAX_TRACER_SIZE;
4185 if (copy_from_user(&buf, ubuf, cnt))
4190 /* strip ending whitespace. */
4191 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4194 err = tracing_set_tracer(tr, buf);
4204 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4205 size_t cnt, loff_t *ppos)
4207 unsigned long *ptr = filp->private_data;
4211 r = snprintf(buf, sizeof(buf), "%ld\n",
4212 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4213 if (r > sizeof(buf))
4215 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4219 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4220 size_t cnt, loff_t *ppos)
4222 unsigned long *ptr = filp->private_data;
4226 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4235 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4237 struct trace_array *tr = inode->i_private;
4238 struct trace_iterator *iter;
4241 if (tracing_disabled)
4244 if (trace_array_get(tr) < 0)
4247 mutex_lock(&trace_types_lock);
4249 /* create a buffer to store the information to pass to userspace */
4250 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4253 __trace_array_put(tr);
4258 * We make a copy of the current tracer to avoid concurrent
4259 * changes on it while we are reading.
4261 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4266 *iter->trace = *tr->current_trace;
4268 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4273 /* trace pipe does not show start of buffer */
4274 cpumask_setall(iter->started);
4276 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4277 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4279 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4280 if (trace_clocks[tr->clock_id].in_ns)
4281 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4284 iter->trace_buffer = &tr->trace_buffer;
4285 iter->cpu_file = tracing_get_cpu(inode);
4286 mutex_init(&iter->mutex);
4287 filp->private_data = iter;
4289 if (iter->trace->pipe_open)
4290 iter->trace->pipe_open(iter);
4292 nonseekable_open(inode, filp);
4294 mutex_unlock(&trace_types_lock);
4300 __trace_array_put(tr);
4301 mutex_unlock(&trace_types_lock);
4305 static int tracing_release_pipe(struct inode *inode, struct file *file)
4307 struct trace_iterator *iter = file->private_data;
4308 struct trace_array *tr = inode->i_private;
4310 mutex_lock(&trace_types_lock);
4312 if (iter->trace->pipe_close)
4313 iter->trace->pipe_close(iter);
4315 mutex_unlock(&trace_types_lock);
4317 free_cpumask_var(iter->started);
4318 mutex_destroy(&iter->mutex);
4322 trace_array_put(tr);
4328 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4330 /* Iterators are static, they should be filled or empty */
4331 if (trace_buffer_iter(iter, iter->cpu_file))
4332 return POLLIN | POLLRDNORM;
4334 if (trace_flags & TRACE_ITER_BLOCK)
4336 * Always select as readable when in blocking mode
4338 return POLLIN | POLLRDNORM;
4340 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4345 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4347 struct trace_iterator *iter = filp->private_data;
4349 return trace_poll(iter, filp, poll_table);
4352 /* Must be called with trace_types_lock mutex held. */
4353 static int tracing_wait_pipe(struct file *filp)
4355 struct trace_iterator *iter = filp->private_data;
4358 while (trace_empty(iter)) {
4360 if ((filp->f_flags & O_NONBLOCK)) {
4365 * We block until we read something and tracing is disabled.
4366 * We still block if tracing is disabled, but we have never
4367 * read anything. This allows a user to cat this file, and
4368 * then enable tracing. But after we have read something,
4369 * we give an EOF when tracing is again disabled.
4371 * iter->pos will be 0 if we haven't read anything.
4373 if (!tracing_is_on() && iter->pos)
4376 mutex_unlock(&iter->mutex);
4378 ret = wait_on_pipe(iter);
4380 mutex_lock(&iter->mutex);
4385 if (signal_pending(current))
4396 tracing_read_pipe(struct file *filp, char __user *ubuf,
4397 size_t cnt, loff_t *ppos)
4399 struct trace_iterator *iter = filp->private_data;
4400 struct trace_array *tr = iter->tr;
4403 /* return any leftover data */
4404 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4408 trace_seq_init(&iter->seq);
4410 /* copy the tracer to avoid using a global lock all around */
4411 mutex_lock(&trace_types_lock);
4412 if (unlikely(iter->trace->name != tr->current_trace->name))
4413 *iter->trace = *tr->current_trace;
4414 mutex_unlock(&trace_types_lock);
4417 * Avoid more than one consumer on a single file descriptor
4418 * This is just a matter of traces coherency, the ring buffer itself
4421 mutex_lock(&iter->mutex);
4422 if (iter->trace->read) {
4423 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4429 sret = tracing_wait_pipe(filp);
4433 /* stop when tracing is finished */
4434 if (trace_empty(iter)) {
4439 if (cnt >= PAGE_SIZE)
4440 cnt = PAGE_SIZE - 1;
4442 /* reset all but tr, trace, and overruns */
4443 memset(&iter->seq, 0,
4444 sizeof(struct trace_iterator) -
4445 offsetof(struct trace_iterator, seq));
4446 cpumask_clear(iter->started);
4449 trace_event_read_lock();
4450 trace_access_lock(iter->cpu_file);
4451 while (trace_find_next_entry_inc(iter) != NULL) {
4452 enum print_line_t ret;
4453 int len = iter->seq.len;
4455 ret = print_trace_line(iter);
4456 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4457 /* don't print partial lines */
4458 iter->seq.len = len;
4461 if (ret != TRACE_TYPE_NO_CONSUME)
4462 trace_consume(iter);
4464 if (iter->seq.len >= cnt)
4468 * Setting the full flag means we reached the trace_seq buffer
4469 * size and we should leave by partial output condition above.
4470 * One of the trace_seq_* functions is not used properly.
4472 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4475 trace_access_unlock(iter->cpu_file);
4476 trace_event_read_unlock();
4478 /* Now copy what we have to the user */
4479 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4480 if (iter->seq.readpos >= iter->seq.len)
4481 trace_seq_init(&iter->seq);
4484 * If there was nothing to send to user, in spite of consuming trace
4485 * entries, go back to wait for more entries.
4491 mutex_unlock(&iter->mutex);
4496 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4499 __free_page(spd->pages[idx]);
4502 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4504 .confirm = generic_pipe_buf_confirm,
4505 .release = generic_pipe_buf_release,
4506 .steal = generic_pipe_buf_steal,
4507 .get = generic_pipe_buf_get,
4511 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4516 /* Seq buffer is page-sized, exactly what we need. */
4518 count = iter->seq.len;
4519 ret = print_trace_line(iter);
4520 count = iter->seq.len - count;
4523 iter->seq.len -= count;
4526 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4527 iter->seq.len -= count;
4531 if (ret != TRACE_TYPE_NO_CONSUME)
4532 trace_consume(iter);
4534 if (!trace_find_next_entry_inc(iter)) {
4544 static ssize_t tracing_splice_read_pipe(struct file *filp,
4546 struct pipe_inode_info *pipe,
4550 struct page *pages_def[PIPE_DEF_BUFFERS];
4551 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4552 struct trace_iterator *iter = filp->private_data;
4553 struct splice_pipe_desc spd = {
4555 .partial = partial_def,
4556 .nr_pages = 0, /* This gets updated below. */
4557 .nr_pages_max = PIPE_DEF_BUFFERS,
4559 .ops = &tracing_pipe_buf_ops,
4560 .spd_release = tracing_spd_release_pipe,
4562 struct trace_array *tr = iter->tr;
4567 if (splice_grow_spd(pipe, &spd))
4570 /* copy the tracer to avoid using a global lock all around */
4571 mutex_lock(&trace_types_lock);
4572 if (unlikely(iter->trace->name != tr->current_trace->name))
4573 *iter->trace = *tr->current_trace;
4574 mutex_unlock(&trace_types_lock);
4576 mutex_lock(&iter->mutex);
4578 if (iter->trace->splice_read) {
4579 ret = iter->trace->splice_read(iter, filp,
4580 ppos, pipe, len, flags);
4585 ret = tracing_wait_pipe(filp);
4589 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4594 trace_event_read_lock();
4595 trace_access_lock(iter->cpu_file);
4597 /* Fill as many pages as possible. */
4598 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4599 spd.pages[i] = alloc_page(GFP_KERNEL);
4603 rem = tracing_fill_pipe_page(rem, iter);
4605 /* Copy the data into the page, so we can start over. */
4606 ret = trace_seq_to_buffer(&iter->seq,
4607 page_address(spd.pages[i]),
4610 __free_page(spd.pages[i]);
4613 spd.partial[i].offset = 0;
4614 spd.partial[i].len = iter->seq.len;
4616 trace_seq_init(&iter->seq);
4619 trace_access_unlock(iter->cpu_file);
4620 trace_event_read_unlock();
4621 mutex_unlock(&iter->mutex);
4625 ret = splice_to_pipe(pipe, &spd);
4627 splice_shrink_spd(&spd);
4631 mutex_unlock(&iter->mutex);
4636 tracing_entries_read(struct file *filp, char __user *ubuf,
4637 size_t cnt, loff_t *ppos)
4639 struct inode *inode = file_inode(filp);
4640 struct trace_array *tr = inode->i_private;
4641 int cpu = tracing_get_cpu(inode);
4646 mutex_lock(&trace_types_lock);
4648 if (cpu == RING_BUFFER_ALL_CPUS) {
4649 int cpu, buf_size_same;
4654 /* check if all cpu sizes are same */
4655 for_each_tracing_cpu(cpu) {
4656 /* fill in the size from first enabled cpu */
4658 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4659 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4665 if (buf_size_same) {
4666 if (!ring_buffer_expanded)
4667 r = sprintf(buf, "%lu (expanded: %lu)\n",
4669 trace_buf_size >> 10);
4671 r = sprintf(buf, "%lu\n", size >> 10);
4673 r = sprintf(buf, "X\n");
4675 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4677 mutex_unlock(&trace_types_lock);
4679 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4684 tracing_entries_write(struct file *filp, const char __user *ubuf,
4685 size_t cnt, loff_t *ppos)
4687 struct inode *inode = file_inode(filp);
4688 struct trace_array *tr = inode->i_private;
4692 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4696 /* must have at least 1 entry */
4700 /* value is in KB */
4702 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4712 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4713 size_t cnt, loff_t *ppos)
4715 struct trace_array *tr = filp->private_data;
4718 unsigned long size = 0, expanded_size = 0;
4720 mutex_lock(&trace_types_lock);
4721 for_each_tracing_cpu(cpu) {
4722 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4723 if (!ring_buffer_expanded)
4724 expanded_size += trace_buf_size >> 10;
4726 if (ring_buffer_expanded)
4727 r = sprintf(buf, "%lu\n", size);
4729 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4730 mutex_unlock(&trace_types_lock);
4732 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4736 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4737 size_t cnt, loff_t *ppos)
4740 * There is no need to read what the user has written, this function
4741 * is just to make sure that there is no error when "echo" is used
4750 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4752 struct trace_array *tr = inode->i_private;
4754 /* disable tracing ? */
4755 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4756 tracer_tracing_off(tr);
4757 /* resize the ring buffer to 0 */
4758 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4760 trace_array_put(tr);
4766 tracing_mark_write(struct file *filp, const char __user *ubuf,
4767 size_t cnt, loff_t *fpos)
4769 unsigned long addr = (unsigned long)ubuf;
4770 struct trace_array *tr = filp->private_data;
4771 struct ring_buffer_event *event;
4772 struct ring_buffer *buffer;
4773 struct print_entry *entry;
4774 unsigned long irq_flags;
4775 struct page *pages[2];
4785 if (tracing_disabled)
4788 if (!(trace_flags & TRACE_ITER_MARKERS))
4791 if (cnt > TRACE_BUF_SIZE)
4792 cnt = TRACE_BUF_SIZE;
4795 * Userspace is injecting traces into the kernel trace buffer.
4796 * We want to be as non intrusive as possible.
4797 * To do so, we do not want to allocate any special buffers
4798 * or take any locks, but instead write the userspace data
4799 * straight into the ring buffer.
4801 * First we need to pin the userspace buffer into memory,
4802 * which, most likely it is, because it just referenced it.
4803 * But there's no guarantee that it is. By using get_user_pages_fast()
4804 * and kmap_atomic/kunmap_atomic() we can get access to the
4805 * pages directly. We then write the data directly into the
4808 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4810 /* check if we cross pages */
4811 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4814 offset = addr & (PAGE_SIZE - 1);
4817 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4818 if (ret < nr_pages) {
4820 put_page(pages[ret]);
4825 for (i = 0; i < nr_pages; i++)
4826 map_page[i] = kmap_atomic(pages[i]);
4828 local_save_flags(irq_flags);
4829 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4830 buffer = tr->trace_buffer.buffer;
4831 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4832 irq_flags, preempt_count());
4834 /* Ring buffer disabled, return as if not open for write */
4839 entry = ring_buffer_event_data(event);
4840 entry->ip = _THIS_IP_;
4842 if (nr_pages == 2) {
4843 len = PAGE_SIZE - offset;
4844 memcpy(&entry->buf, map_page[0] + offset, len);
4845 memcpy(&entry->buf[len], map_page[1], cnt - len);
4847 memcpy(&entry->buf, map_page[0] + offset, cnt);
4849 if (entry->buf[cnt - 1] != '\n') {
4850 entry->buf[cnt] = '\n';
4851 entry->buf[cnt + 1] = '\0';
4853 entry->buf[cnt] = '\0';
4855 __buffer_unlock_commit(buffer, event);
4862 for (i = 0; i < nr_pages; i++){
4863 kunmap_atomic(map_page[i]);
4870 static int tracing_clock_show(struct seq_file *m, void *v)
4872 struct trace_array *tr = m->private;
4875 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4877 "%s%s%s%s", i ? " " : "",
4878 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4879 i == tr->clock_id ? "]" : "");
4885 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4889 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4890 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4893 if (i == ARRAY_SIZE(trace_clocks))
4896 mutex_lock(&trace_types_lock);
4900 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4903 * New clock may not be consistent with the previous clock.
4904 * Reset the buffer so that it doesn't have incomparable timestamps.
4906 tracing_reset_online_cpus(&tr->trace_buffer);
4908 #ifdef CONFIG_TRACER_MAX_TRACE
4909 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4910 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4911 tracing_reset_online_cpus(&tr->max_buffer);
4914 mutex_unlock(&trace_types_lock);
4919 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4920 size_t cnt, loff_t *fpos)
4922 struct seq_file *m = filp->private_data;
4923 struct trace_array *tr = m->private;
4925 const char *clockstr;
4928 if (cnt >= sizeof(buf))
4931 if (copy_from_user(&buf, ubuf, cnt))
4936 clockstr = strstrip(buf);
4938 ret = tracing_set_clock(tr, clockstr);
4947 static int tracing_clock_open(struct inode *inode, struct file *file)
4949 struct trace_array *tr = inode->i_private;
4952 if (tracing_disabled)
4955 if (trace_array_get(tr))
4958 ret = single_open(file, tracing_clock_show, inode->i_private);
4960 trace_array_put(tr);
4965 struct ftrace_buffer_info {
4966 struct trace_iterator iter;
4971 #ifdef CONFIG_TRACER_SNAPSHOT
4972 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4974 struct trace_array *tr = inode->i_private;
4975 struct trace_iterator *iter;
4979 if (trace_array_get(tr) < 0)
4982 if (file->f_mode & FMODE_READ) {
4983 iter = __tracing_open(inode, file, true);
4985 ret = PTR_ERR(iter);
4987 /* Writes still need the seq_file to hold the private data */
4989 m = kzalloc(sizeof(*m), GFP_KERNEL);
4992 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5000 iter->trace_buffer = &tr->max_buffer;
5001 iter->cpu_file = tracing_get_cpu(inode);
5003 file->private_data = m;
5007 trace_array_put(tr);
5013 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5016 struct seq_file *m = filp->private_data;
5017 struct trace_iterator *iter = m->private;
5018 struct trace_array *tr = iter->tr;
5022 ret = tracing_update_buffers();
5026 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5030 mutex_lock(&trace_types_lock);
5032 if (tr->current_trace->use_max_tr) {
5039 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5043 if (tr->allocated_snapshot)
5047 /* Only allow per-cpu swap if the ring buffer supports it */
5048 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5049 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5054 if (!tr->allocated_snapshot) {
5055 ret = alloc_snapshot(tr);
5059 local_irq_disable();
5060 /* Now, we're going to swap */
5061 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5062 update_max_tr(tr, current, smp_processor_id());
5064 update_max_tr_single(tr, current, iter->cpu_file);
5068 if (tr->allocated_snapshot) {
5069 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5070 tracing_reset_online_cpus(&tr->max_buffer);
5072 tracing_reset(&tr->max_buffer, iter->cpu_file);
5082 mutex_unlock(&trace_types_lock);
5086 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5088 struct seq_file *m = file->private_data;
5091 ret = tracing_release(inode, file);
5093 if (file->f_mode & FMODE_READ)
5096 /* If write only, the seq_file is just a stub */
5104 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5105 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5106 size_t count, loff_t *ppos);
5107 static int tracing_buffers_release(struct inode *inode, struct file *file);
5108 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5109 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5111 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5113 struct ftrace_buffer_info *info;
5116 ret = tracing_buffers_open(inode, filp);
5120 info = filp->private_data;
5122 if (info->iter.trace->use_max_tr) {
5123 tracing_buffers_release(inode, filp);
5127 info->iter.snapshot = true;
5128 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5133 #endif /* CONFIG_TRACER_SNAPSHOT */
5136 static const struct file_operations tracing_max_lat_fops = {
5137 .open = tracing_open_generic,
5138 .read = tracing_max_lat_read,
5139 .write = tracing_max_lat_write,
5140 .llseek = generic_file_llseek,
5143 static const struct file_operations set_tracer_fops = {
5144 .open = tracing_open_generic,
5145 .read = tracing_set_trace_read,
5146 .write = tracing_set_trace_write,
5147 .llseek = generic_file_llseek,
5150 static const struct file_operations tracing_pipe_fops = {
5151 .open = tracing_open_pipe,
5152 .poll = tracing_poll_pipe,
5153 .read = tracing_read_pipe,
5154 .splice_read = tracing_splice_read_pipe,
5155 .release = tracing_release_pipe,
5156 .llseek = no_llseek,
5159 static const struct file_operations tracing_entries_fops = {
5160 .open = tracing_open_generic_tr,
5161 .read = tracing_entries_read,
5162 .write = tracing_entries_write,
5163 .llseek = generic_file_llseek,
5164 .release = tracing_release_generic_tr,
5167 static const struct file_operations tracing_total_entries_fops = {
5168 .open = tracing_open_generic_tr,
5169 .read = tracing_total_entries_read,
5170 .llseek = generic_file_llseek,
5171 .release = tracing_release_generic_tr,
5174 static const struct file_operations tracing_free_buffer_fops = {
5175 .open = tracing_open_generic_tr,
5176 .write = tracing_free_buffer_write,
5177 .release = tracing_free_buffer_release,
5180 static const struct file_operations tracing_mark_fops = {
5181 .open = tracing_open_generic_tr,
5182 .write = tracing_mark_write,
5183 .llseek = generic_file_llseek,
5184 .release = tracing_release_generic_tr,
5187 static const struct file_operations trace_clock_fops = {
5188 .open = tracing_clock_open,
5190 .llseek = seq_lseek,
5191 .release = tracing_single_release_tr,
5192 .write = tracing_clock_write,
5195 #ifdef CONFIG_TRACER_SNAPSHOT
5196 static const struct file_operations snapshot_fops = {
5197 .open = tracing_snapshot_open,
5199 .write = tracing_snapshot_write,
5200 .llseek = tracing_lseek,
5201 .release = tracing_snapshot_release,
5204 static const struct file_operations snapshot_raw_fops = {
5205 .open = snapshot_raw_open,
5206 .read = tracing_buffers_read,
5207 .release = tracing_buffers_release,
5208 .splice_read = tracing_buffers_splice_read,
5209 .llseek = no_llseek,
5212 #endif /* CONFIG_TRACER_SNAPSHOT */
5214 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5216 struct trace_array *tr = inode->i_private;
5217 struct ftrace_buffer_info *info;
5220 if (tracing_disabled)
5223 if (trace_array_get(tr) < 0)
5226 info = kzalloc(sizeof(*info), GFP_KERNEL);
5228 trace_array_put(tr);
5232 mutex_lock(&trace_types_lock);
5235 info->iter.cpu_file = tracing_get_cpu(inode);
5236 info->iter.trace = tr->current_trace;
5237 info->iter.trace_buffer = &tr->trace_buffer;
5239 /* Force reading ring buffer for first read */
5240 info->read = (unsigned int)-1;
5242 filp->private_data = info;
5244 mutex_unlock(&trace_types_lock);
5246 ret = nonseekable_open(inode, filp);
5248 trace_array_put(tr);
5254 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5256 struct ftrace_buffer_info *info = filp->private_data;
5257 struct trace_iterator *iter = &info->iter;
5259 return trace_poll(iter, filp, poll_table);
5263 tracing_buffers_read(struct file *filp, char __user *ubuf,
5264 size_t count, loff_t *ppos)
5266 struct ftrace_buffer_info *info = filp->private_data;
5267 struct trace_iterator *iter = &info->iter;
5274 mutex_lock(&trace_types_lock);
5276 #ifdef CONFIG_TRACER_MAX_TRACE
5277 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5284 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5290 /* Do we have previous read data to read? */
5291 if (info->read < PAGE_SIZE)
5295 trace_access_lock(iter->cpu_file);
5296 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5300 trace_access_unlock(iter->cpu_file);
5303 if (trace_empty(iter)) {
5304 if ((filp->f_flags & O_NONBLOCK)) {
5308 mutex_unlock(&trace_types_lock);
5309 ret = wait_on_pipe(iter);
5310 mutex_lock(&trace_types_lock);
5315 if (signal_pending(current)) {
5327 size = PAGE_SIZE - info->read;
5331 ret = copy_to_user(ubuf, info->spare + info->read, size);
5342 mutex_unlock(&trace_types_lock);
5347 static int tracing_buffers_release(struct inode *inode, struct file *file)
5349 struct ftrace_buffer_info *info = file->private_data;
5350 struct trace_iterator *iter = &info->iter;
5352 mutex_lock(&trace_types_lock);
5354 __trace_array_put(iter->tr);
5357 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5360 mutex_unlock(&trace_types_lock);
5366 struct ring_buffer *buffer;
5371 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5372 struct pipe_buffer *buf)
5374 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5379 ring_buffer_free_read_page(ref->buffer, ref->page);
5384 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5385 struct pipe_buffer *buf)
5387 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5392 /* Pipe buffer operations for a buffer. */
5393 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5395 .confirm = generic_pipe_buf_confirm,
5396 .release = buffer_pipe_buf_release,
5397 .steal = generic_pipe_buf_steal,
5398 .get = buffer_pipe_buf_get,
5402 * Callback from splice_to_pipe(), if we need to release some pages
5403 * at the end of the spd in case we error'ed out in filling the pipe.
5405 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5407 struct buffer_ref *ref =
5408 (struct buffer_ref *)spd->partial[i].private;
5413 ring_buffer_free_read_page(ref->buffer, ref->page);
5415 spd->partial[i].private = 0;
5419 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5420 struct pipe_inode_info *pipe, size_t len,
5423 struct ftrace_buffer_info *info = file->private_data;
5424 struct trace_iterator *iter = &info->iter;
5425 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5426 struct page *pages_def[PIPE_DEF_BUFFERS];
5427 struct splice_pipe_desc spd = {
5429 .partial = partial_def,
5430 .nr_pages_max = PIPE_DEF_BUFFERS,
5432 .ops = &buffer_pipe_buf_ops,
5433 .spd_release = buffer_spd_release,
5435 struct buffer_ref *ref;
5436 int entries, size, i;
5439 mutex_lock(&trace_types_lock);
5441 #ifdef CONFIG_TRACER_MAX_TRACE
5442 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5448 if (splice_grow_spd(pipe, &spd)) {
5453 if (*ppos & (PAGE_SIZE - 1)) {
5458 if (len & (PAGE_SIZE - 1)) {
5459 if (len < PAGE_SIZE) {
5467 trace_access_lock(iter->cpu_file);
5468 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5470 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5474 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5479 ref->buffer = iter->trace_buffer->buffer;
5480 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5486 r = ring_buffer_read_page(ref->buffer, &ref->page,
5487 len, iter->cpu_file, 1);
5489 ring_buffer_free_read_page(ref->buffer, ref->page);
5495 * zero out any left over data, this is going to
5498 size = ring_buffer_page_len(ref->page);
5499 if (size < PAGE_SIZE)
5500 memset(ref->page + size, 0, PAGE_SIZE - size);
5502 page = virt_to_page(ref->page);
5504 spd.pages[i] = page;
5505 spd.partial[i].len = PAGE_SIZE;
5506 spd.partial[i].offset = 0;
5507 spd.partial[i].private = (unsigned long)ref;
5511 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5514 trace_access_unlock(iter->cpu_file);
5517 /* did we read anything? */
5518 if (!spd.nr_pages) {
5519 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5523 mutex_unlock(&trace_types_lock);
5524 ret = wait_on_pipe(iter);
5525 mutex_lock(&trace_types_lock);
5528 if (signal_pending(current)) {
5535 ret = splice_to_pipe(pipe, &spd);
5536 splice_shrink_spd(&spd);
5538 mutex_unlock(&trace_types_lock);
5543 static const struct file_operations tracing_buffers_fops = {
5544 .open = tracing_buffers_open,
5545 .read = tracing_buffers_read,
5546 .poll = tracing_buffers_poll,
5547 .release = tracing_buffers_release,
5548 .splice_read = tracing_buffers_splice_read,
5549 .llseek = no_llseek,
5553 tracing_stats_read(struct file *filp, char __user *ubuf,
5554 size_t count, loff_t *ppos)
5556 struct inode *inode = file_inode(filp);
5557 struct trace_array *tr = inode->i_private;
5558 struct trace_buffer *trace_buf = &tr->trace_buffer;
5559 int cpu = tracing_get_cpu(inode);
5560 struct trace_seq *s;
5562 unsigned long long t;
5563 unsigned long usec_rem;
5565 s = kmalloc(sizeof(*s), GFP_KERNEL);
5571 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5572 trace_seq_printf(s, "entries: %ld\n", cnt);
5574 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5575 trace_seq_printf(s, "overrun: %ld\n", cnt);
5577 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5578 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5580 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5581 trace_seq_printf(s, "bytes: %ld\n", cnt);
5583 if (trace_clocks[tr->clock_id].in_ns) {
5584 /* local or global for trace_clock */
5585 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5586 usec_rem = do_div(t, USEC_PER_SEC);
5587 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5590 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5591 usec_rem = do_div(t, USEC_PER_SEC);
5592 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5594 /* counter or tsc mode for trace_clock */
5595 trace_seq_printf(s, "oldest event ts: %llu\n",
5596 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5598 trace_seq_printf(s, "now ts: %llu\n",
5599 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5602 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5603 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5605 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5606 trace_seq_printf(s, "read events: %ld\n", cnt);
5608 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5615 static const struct file_operations tracing_stats_fops = {
5616 .open = tracing_open_generic_tr,
5617 .read = tracing_stats_read,
5618 .llseek = generic_file_llseek,
5619 .release = tracing_release_generic_tr,
5622 #ifdef CONFIG_DYNAMIC_FTRACE
5624 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5630 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5631 size_t cnt, loff_t *ppos)
5633 static char ftrace_dyn_info_buffer[1024];
5634 static DEFINE_MUTEX(dyn_info_mutex);
5635 unsigned long *p = filp->private_data;
5636 char *buf = ftrace_dyn_info_buffer;
5637 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5640 mutex_lock(&dyn_info_mutex);
5641 r = sprintf(buf, "%ld ", *p);
5643 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5646 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5648 mutex_unlock(&dyn_info_mutex);
5653 static const struct file_operations tracing_dyn_info_fops = {
5654 .open = tracing_open_generic,
5655 .read = tracing_read_dyn_info,
5656 .llseek = generic_file_llseek,
5658 #endif /* CONFIG_DYNAMIC_FTRACE */
5660 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5662 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5668 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5670 unsigned long *count = (long *)data;
5682 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5683 struct ftrace_probe_ops *ops, void *data)
5685 long count = (long)data;
5687 seq_printf(m, "%ps:", (void *)ip);
5689 seq_printf(m, "snapshot");
5692 seq_printf(m, ":unlimited\n");
5694 seq_printf(m, ":count=%ld\n", count);
5699 static struct ftrace_probe_ops snapshot_probe_ops = {
5700 .func = ftrace_snapshot,
5701 .print = ftrace_snapshot_print,
5704 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5705 .func = ftrace_count_snapshot,
5706 .print = ftrace_snapshot_print,
5710 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5711 char *glob, char *cmd, char *param, int enable)
5713 struct ftrace_probe_ops *ops;
5714 void *count = (void *)-1;
5718 /* hash funcs only work with set_ftrace_filter */
5722 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5724 if (glob[0] == '!') {
5725 unregister_ftrace_function_probe_func(glob+1, ops);
5732 number = strsep(¶m, ":");
5734 if (!strlen(number))
5738 * We use the callback data field (which is a pointer)
5741 ret = kstrtoul(number, 0, (unsigned long *)&count);
5746 ret = register_ftrace_function_probe(glob, ops, count);
5749 alloc_snapshot(&global_trace);
5751 return ret < 0 ? ret : 0;
5754 static struct ftrace_func_command ftrace_snapshot_cmd = {
5756 .func = ftrace_trace_snapshot_callback,
5759 static __init int register_snapshot_cmd(void)
5761 return register_ftrace_command(&ftrace_snapshot_cmd);
5764 static inline __init int register_snapshot_cmd(void) { return 0; }
5765 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5767 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5772 if (!debugfs_initialized())
5775 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5776 tr->dir = debugfs_create_dir("tracing", NULL);
5779 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5784 struct dentry *tracing_init_dentry(void)
5786 return tracing_init_dentry_tr(&global_trace);
5789 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5791 struct dentry *d_tracer;
5794 return tr->percpu_dir;
5796 d_tracer = tracing_init_dentry_tr(tr);
5800 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5802 WARN_ONCE(!tr->percpu_dir,
5803 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5805 return tr->percpu_dir;
5808 static struct dentry *
5809 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5810 void *data, long cpu, const struct file_operations *fops)
5812 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5814 if (ret) /* See tracing_get_cpu() */
5815 ret->d_inode->i_cdev = (void *)(cpu + 1);
5820 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5822 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5823 struct dentry *d_cpu;
5824 char cpu_dir[30]; /* 30 characters should be more than enough */
5829 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5830 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5832 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5836 /* per cpu trace_pipe */
5837 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5838 tr, cpu, &tracing_pipe_fops);
5841 trace_create_cpu_file("trace", 0644, d_cpu,
5842 tr, cpu, &tracing_fops);
5844 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5845 tr, cpu, &tracing_buffers_fops);
5847 trace_create_cpu_file("stats", 0444, d_cpu,
5848 tr, cpu, &tracing_stats_fops);
5850 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5851 tr, cpu, &tracing_entries_fops);
5853 #ifdef CONFIG_TRACER_SNAPSHOT
5854 trace_create_cpu_file("snapshot", 0644, d_cpu,
5855 tr, cpu, &snapshot_fops);
5857 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5858 tr, cpu, &snapshot_raw_fops);
5862 #ifdef CONFIG_FTRACE_SELFTEST
5863 /* Let selftest have access to static functions in this file */
5864 #include "trace_selftest.c"
5867 struct trace_option_dentry {
5868 struct tracer_opt *opt;
5869 struct tracer_flags *flags;
5870 struct trace_array *tr;
5871 struct dentry *entry;
5875 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5878 struct trace_option_dentry *topt = filp->private_data;
5881 if (topt->flags->val & topt->opt->bit)
5886 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5890 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5893 struct trace_option_dentry *topt = filp->private_data;
5897 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5901 if (val != 0 && val != 1)
5904 if (!!(topt->flags->val & topt->opt->bit) != val) {
5905 mutex_lock(&trace_types_lock);
5906 ret = __set_tracer_option(topt->tr, topt->flags,
5908 mutex_unlock(&trace_types_lock);
5919 static const struct file_operations trace_options_fops = {
5920 .open = tracing_open_generic,
5921 .read = trace_options_read,
5922 .write = trace_options_write,
5923 .llseek = generic_file_llseek,
5927 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5930 long index = (long)filp->private_data;
5933 if (trace_flags & (1 << index))
5938 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5942 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5945 struct trace_array *tr = &global_trace;
5946 long index = (long)filp->private_data;
5950 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5954 if (val != 0 && val != 1)
5957 mutex_lock(&trace_types_lock);
5958 ret = set_tracer_flag(tr, 1 << index, val);
5959 mutex_unlock(&trace_types_lock);
5969 static const struct file_operations trace_options_core_fops = {
5970 .open = tracing_open_generic,
5971 .read = trace_options_core_read,
5972 .write = trace_options_core_write,
5973 .llseek = generic_file_llseek,
5976 struct dentry *trace_create_file(const char *name,
5978 struct dentry *parent,
5980 const struct file_operations *fops)
5984 ret = debugfs_create_file(name, mode, parent, data, fops);
5986 pr_warning("Could not create debugfs '%s' entry\n", name);
5992 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5994 struct dentry *d_tracer;
5999 d_tracer = tracing_init_dentry_tr(tr);
6003 tr->options = debugfs_create_dir("options", d_tracer);
6005 pr_warning("Could not create debugfs directory 'options'\n");
6013 create_trace_option_file(struct trace_array *tr,
6014 struct trace_option_dentry *topt,
6015 struct tracer_flags *flags,
6016 struct tracer_opt *opt)
6018 struct dentry *t_options;
6020 t_options = trace_options_init_dentry(tr);
6024 topt->flags = flags;
6028 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6029 &trace_options_fops);
6033 static struct trace_option_dentry *
6034 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6036 struct trace_option_dentry *topts;
6037 struct tracer_flags *flags;
6038 struct tracer_opt *opts;
6044 flags = tracer->flags;
6046 if (!flags || !flags->opts)
6051 for (cnt = 0; opts[cnt].name; cnt++)
6054 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6058 for (cnt = 0; opts[cnt].name; cnt++)
6059 create_trace_option_file(tr, &topts[cnt], flags,
6066 destroy_trace_option_files(struct trace_option_dentry *topts)
6073 for (cnt = 0; topts[cnt].opt; cnt++)
6074 debugfs_remove(topts[cnt].entry);
6079 static struct dentry *
6080 create_trace_option_core_file(struct trace_array *tr,
6081 const char *option, long index)
6083 struct dentry *t_options;
6085 t_options = trace_options_init_dentry(tr);
6089 return trace_create_file(option, 0644, t_options, (void *)index,
6090 &trace_options_core_fops);
6093 static __init void create_trace_options_dir(struct trace_array *tr)
6095 struct dentry *t_options;
6098 t_options = trace_options_init_dentry(tr);
6102 for (i = 0; trace_options[i]; i++)
6103 create_trace_option_core_file(tr, trace_options[i], i);
6107 rb_simple_read(struct file *filp, char __user *ubuf,
6108 size_t cnt, loff_t *ppos)
6110 struct trace_array *tr = filp->private_data;
6114 r = tracer_tracing_is_on(tr);
6115 r = sprintf(buf, "%d\n", r);
6117 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6121 rb_simple_write(struct file *filp, const char __user *ubuf,
6122 size_t cnt, loff_t *ppos)
6124 struct trace_array *tr = filp->private_data;
6125 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6129 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6134 mutex_lock(&trace_types_lock);
6136 tracer_tracing_on(tr);
6137 if (tr->current_trace->start)
6138 tr->current_trace->start(tr);
6140 tracer_tracing_off(tr);
6141 if (tr->current_trace->stop)
6142 tr->current_trace->stop(tr);
6144 mutex_unlock(&trace_types_lock);
6152 static const struct file_operations rb_simple_fops = {
6153 .open = tracing_open_generic_tr,
6154 .read = rb_simple_read,
6155 .write = rb_simple_write,
6156 .release = tracing_release_generic_tr,
6157 .llseek = default_llseek,
6160 struct dentry *trace_instance_dir;
6163 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6166 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6168 enum ring_buffer_flags rb_flags;
6170 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6174 buf->buffer = ring_buffer_alloc(size, rb_flags);
6178 buf->data = alloc_percpu(struct trace_array_cpu);
6180 ring_buffer_free(buf->buffer);
6184 /* Allocate the first page for all buffers */
6185 set_buffer_entries(&tr->trace_buffer,
6186 ring_buffer_size(tr->trace_buffer.buffer, 0));
6191 static int allocate_trace_buffers(struct trace_array *tr, int size)
6195 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6199 #ifdef CONFIG_TRACER_MAX_TRACE
6200 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6201 allocate_snapshot ? size : 1);
6203 ring_buffer_free(tr->trace_buffer.buffer);
6204 free_percpu(tr->trace_buffer.data);
6207 tr->allocated_snapshot = allocate_snapshot;
6210 * Only the top level trace array gets its snapshot allocated
6211 * from the kernel command line.
6213 allocate_snapshot = false;
6218 static void free_trace_buffer(struct trace_buffer *buf)
6221 ring_buffer_free(buf->buffer);
6223 free_percpu(buf->data);
6228 static void free_trace_buffers(struct trace_array *tr)
6233 free_trace_buffer(&tr->trace_buffer);
6235 #ifdef CONFIG_TRACER_MAX_TRACE
6236 free_trace_buffer(&tr->max_buffer);
6240 static int new_instance_create(const char *name)
6242 struct trace_array *tr;
6245 mutex_lock(&trace_types_lock);
6248 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6249 if (tr->name && strcmp(tr->name, name) == 0)
6254 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6258 tr->name = kstrdup(name, GFP_KERNEL);
6262 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6265 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6267 raw_spin_lock_init(&tr->start_lock);
6269 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6271 tr->current_trace = &nop_trace;
6273 INIT_LIST_HEAD(&tr->systems);
6274 INIT_LIST_HEAD(&tr->events);
6276 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6279 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6283 ret = event_trace_add_tracer(tr->dir, tr);
6285 debugfs_remove_recursive(tr->dir);
6289 init_tracer_debugfs(tr, tr->dir);
6291 list_add(&tr->list, &ftrace_trace_arrays);
6293 mutex_unlock(&trace_types_lock);
6298 free_trace_buffers(tr);
6299 free_cpumask_var(tr->tracing_cpumask);
6304 mutex_unlock(&trace_types_lock);
6310 static int instance_delete(const char *name)
6312 struct trace_array *tr;
6316 mutex_lock(&trace_types_lock);
6319 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6320 if (tr->name && strcmp(tr->name, name) == 0) {
6332 list_del(&tr->list);
6334 tracing_set_nop(tr);
6335 event_trace_del_tracer(tr);
6336 ftrace_destroy_function_files(tr);
6337 debugfs_remove_recursive(tr->dir);
6338 free_trace_buffers(tr);
6346 mutex_unlock(&trace_types_lock);
6351 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6353 struct dentry *parent;
6356 /* Paranoid: Make sure the parent is the "instances" directory */
6357 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6358 if (WARN_ON_ONCE(parent != trace_instance_dir))
6362 * The inode mutex is locked, but debugfs_create_dir() will also
6363 * take the mutex. As the instances directory can not be destroyed
6364 * or changed in any other way, it is safe to unlock it, and
6365 * let the dentry try. If two users try to make the same dir at
6366 * the same time, then the new_instance_create() will determine the
6369 mutex_unlock(&inode->i_mutex);
6371 ret = new_instance_create(dentry->d_iname);
6373 mutex_lock(&inode->i_mutex);
6378 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6380 struct dentry *parent;
6383 /* Paranoid: Make sure the parent is the "instances" directory */
6384 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6385 if (WARN_ON_ONCE(parent != trace_instance_dir))
6388 /* The caller did a dget() on dentry */
6389 mutex_unlock(&dentry->d_inode->i_mutex);
6392 * The inode mutex is locked, but debugfs_create_dir() will also
6393 * take the mutex. As the instances directory can not be destroyed
6394 * or changed in any other way, it is safe to unlock it, and
6395 * let the dentry try. If two users try to make the same dir at
6396 * the same time, then the instance_delete() will determine the
6399 mutex_unlock(&inode->i_mutex);
6401 ret = instance_delete(dentry->d_iname);
6403 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6404 mutex_lock(&dentry->d_inode->i_mutex);
6409 static const struct inode_operations instance_dir_inode_operations = {
6410 .lookup = simple_lookup,
6411 .mkdir = instance_mkdir,
6412 .rmdir = instance_rmdir,
6415 static __init void create_trace_instances(struct dentry *d_tracer)
6417 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6418 if (WARN_ON(!trace_instance_dir))
6421 /* Hijack the dir inode operations, to allow mkdir */
6422 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6426 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6430 trace_create_file("available_tracers", 0444, d_tracer,
6431 tr, &show_traces_fops);
6433 trace_create_file("current_tracer", 0644, d_tracer,
6434 tr, &set_tracer_fops);
6436 trace_create_file("tracing_cpumask", 0644, d_tracer,
6437 tr, &tracing_cpumask_fops);
6439 trace_create_file("trace_options", 0644, d_tracer,
6440 tr, &tracing_iter_fops);
6442 trace_create_file("trace", 0644, d_tracer,
6445 trace_create_file("trace_pipe", 0444, d_tracer,
6446 tr, &tracing_pipe_fops);
6448 trace_create_file("buffer_size_kb", 0644, d_tracer,
6449 tr, &tracing_entries_fops);
6451 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6452 tr, &tracing_total_entries_fops);
6454 trace_create_file("free_buffer", 0200, d_tracer,
6455 tr, &tracing_free_buffer_fops);
6457 trace_create_file("trace_marker", 0220, d_tracer,
6458 tr, &tracing_mark_fops);
6460 trace_create_file("trace_clock", 0644, d_tracer, tr,
6463 trace_create_file("tracing_on", 0644, d_tracer,
6464 tr, &rb_simple_fops);
6466 #ifdef CONFIG_TRACER_MAX_TRACE
6467 trace_create_file("tracing_max_latency", 0644, d_tracer,
6468 &tr->max_latency, &tracing_max_lat_fops);
6471 if (ftrace_create_function_files(tr, d_tracer))
6472 WARN(1, "Could not allocate function filter files");
6474 #ifdef CONFIG_TRACER_SNAPSHOT
6475 trace_create_file("snapshot", 0644, d_tracer,
6476 tr, &snapshot_fops);
6479 for_each_tracing_cpu(cpu)
6480 tracing_init_debugfs_percpu(tr, cpu);
6484 static __init int tracer_init_debugfs(void)
6486 struct dentry *d_tracer;
6488 trace_access_lock_init();
6490 d_tracer = tracing_init_dentry();
6494 init_tracer_debugfs(&global_trace, d_tracer);
6496 trace_create_file("tracing_thresh", 0644, d_tracer,
6497 &tracing_thresh, &tracing_max_lat_fops);
6499 trace_create_file("README", 0444, d_tracer,
6500 NULL, &tracing_readme_fops);
6502 trace_create_file("saved_cmdlines", 0444, d_tracer,
6503 NULL, &tracing_saved_cmdlines_fops);
6505 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6506 NULL, &tracing_saved_cmdlines_size_fops);
6508 #ifdef CONFIG_DYNAMIC_FTRACE
6509 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6510 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6513 create_trace_instances(d_tracer);
6515 create_trace_options_dir(&global_trace);
6520 static int trace_panic_handler(struct notifier_block *this,
6521 unsigned long event, void *unused)
6523 if (ftrace_dump_on_oops)
6524 ftrace_dump(ftrace_dump_on_oops);
6528 static struct notifier_block trace_panic_notifier = {
6529 .notifier_call = trace_panic_handler,
6531 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6534 static int trace_die_handler(struct notifier_block *self,
6540 if (ftrace_dump_on_oops)
6541 ftrace_dump(ftrace_dump_on_oops);
6549 static struct notifier_block trace_die_notifier = {
6550 .notifier_call = trace_die_handler,
6555 * printk is set to max of 1024, we really don't need it that big.
6556 * Nothing should be printing 1000 characters anyway.
6558 #define TRACE_MAX_PRINT 1000
6561 * Define here KERN_TRACE so that we have one place to modify
6562 * it if we decide to change what log level the ftrace dump
6565 #define KERN_TRACE KERN_EMERG
6568 trace_printk_seq(struct trace_seq *s)
6570 /* Probably should print a warning here. */
6571 if (s->len >= TRACE_MAX_PRINT)
6572 s->len = TRACE_MAX_PRINT;
6574 /* should be zero ended, but we are paranoid. */
6575 s->buffer[s->len] = 0;
6577 printk(KERN_TRACE "%s", s->buffer);
6582 void trace_init_global_iter(struct trace_iterator *iter)
6584 iter->tr = &global_trace;
6585 iter->trace = iter->tr->current_trace;
6586 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6587 iter->trace_buffer = &global_trace.trace_buffer;
6589 if (iter->trace && iter->trace->open)
6590 iter->trace->open(iter);
6592 /* Annotate start of buffers if we had overruns */
6593 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6594 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6596 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6597 if (trace_clocks[iter->tr->clock_id].in_ns)
6598 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6601 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6603 /* use static because iter can be a bit big for the stack */
6604 static struct trace_iterator iter;
6605 static atomic_t dump_running;
6606 unsigned int old_userobj;
6607 unsigned long flags;
6610 /* Only allow one dump user at a time. */
6611 if (atomic_inc_return(&dump_running) != 1) {
6612 atomic_dec(&dump_running);
6617 * Always turn off tracing when we dump.
6618 * We don't need to show trace output of what happens
6619 * between multiple crashes.
6621 * If the user does a sysrq-z, then they can re-enable
6622 * tracing with echo 1 > tracing_on.
6626 local_irq_save(flags);
6628 /* Simulate the iterator */
6629 trace_init_global_iter(&iter);
6631 for_each_tracing_cpu(cpu) {
6632 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6635 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6637 /* don't look at user memory in panic mode */
6638 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6640 switch (oops_dump_mode) {
6642 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6645 iter.cpu_file = raw_smp_processor_id();
6650 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6651 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6654 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6656 /* Did function tracer already get disabled? */
6657 if (ftrace_is_dead()) {
6658 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6659 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6663 * We need to stop all tracing on all CPUS to read the
6664 * the next buffer. This is a bit expensive, but is
6665 * not done often. We fill all what we can read,
6666 * and then release the locks again.
6669 while (!trace_empty(&iter)) {
6672 printk(KERN_TRACE "---------------------------------\n");
6676 /* reset all but tr, trace, and overruns */
6677 memset(&iter.seq, 0,
6678 sizeof(struct trace_iterator) -
6679 offsetof(struct trace_iterator, seq));
6680 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6683 if (trace_find_next_entry_inc(&iter) != NULL) {
6686 ret = print_trace_line(&iter);
6687 if (ret != TRACE_TYPE_NO_CONSUME)
6688 trace_consume(&iter);
6690 touch_nmi_watchdog();
6692 trace_printk_seq(&iter.seq);
6696 printk(KERN_TRACE " (ftrace buffer empty)\n");
6698 printk(KERN_TRACE "---------------------------------\n");
6701 trace_flags |= old_userobj;
6703 for_each_tracing_cpu(cpu) {
6704 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6706 atomic_dec(&dump_running);
6707 local_irq_restore(flags);
6709 EXPORT_SYMBOL_GPL(ftrace_dump);
6711 __init static int tracer_alloc_buffers(void)
6717 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6720 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6721 goto out_free_buffer_mask;
6723 /* Only allocate trace_printk buffers if a trace_printk exists */
6724 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6725 /* Must be called before global_trace.buffer is allocated */
6726 trace_printk_init_buffers();
6728 /* To save memory, keep the ring buffer size to its minimum */
6729 if (ring_buffer_expanded)
6730 ring_buf_size = trace_buf_size;
6734 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6735 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6737 raw_spin_lock_init(&global_trace.start_lock);
6739 /* Used for event triggers */
6740 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6742 goto out_free_cpumask;
6744 if (trace_create_savedcmd() < 0)
6745 goto out_free_temp_buffer;
6747 /* TODO: make the number of buffers hot pluggable with CPUS */
6748 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6749 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6751 goto out_free_savedcmd;
6754 if (global_trace.buffer_disabled)
6757 if (trace_boot_clock) {
6758 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6760 pr_warning("Trace clock %s not defined, going back to default\n",
6765 * register_tracer() might reference current_trace, so it
6766 * needs to be set before we register anything. This is
6767 * just a bootstrap of current_trace anyway.
6769 global_trace.current_trace = &nop_trace;
6771 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6773 ftrace_init_global_array_ops(&global_trace);
6775 register_tracer(&nop_trace);
6777 /* All seems OK, enable tracing */
6778 tracing_disabled = 0;
6780 atomic_notifier_chain_register(&panic_notifier_list,
6781 &trace_panic_notifier);
6783 register_die_notifier(&trace_die_notifier);
6785 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6787 INIT_LIST_HEAD(&global_trace.systems);
6788 INIT_LIST_HEAD(&global_trace.events);
6789 list_add(&global_trace.list, &ftrace_trace_arrays);
6791 while (trace_boot_options) {
6794 option = strsep(&trace_boot_options, ",");
6795 trace_set_options(&global_trace, option);
6798 register_snapshot_cmd();
6803 free_saved_cmdlines_buffer(savedcmd);
6804 out_free_temp_buffer:
6805 ring_buffer_free(temp_buffer);
6807 free_cpumask_var(global_trace.tracing_cpumask);
6808 out_free_buffer_mask:
6809 free_cpumask_var(tracing_buffer_mask);
6814 __init static int clear_boot_tracer(void)
6817 * The default tracer at boot buffer is an init section.
6818 * This function is called in lateinit. If we did not
6819 * find the boot tracer, then clear it out, to prevent
6820 * later registration from accessing the buffer that is
6821 * about to be freed.
6823 if (!default_bootup_tracer)
6826 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6827 default_bootup_tracer);
6828 default_bootup_tracer = NULL;
6833 early_initcall(tracer_alloc_buffers);
6834 fs_initcall(tracer_init_debugfs);
6835 late_initcall(clear_boot_tracer);