]> git.karo-electronics.de Git - karo-tx-linux.git/blobdiff - tools/perf/builtin-trace.c
perf trace: Add beautifier for futex 'operation' parm
[karo-tx-linux.git] / tools / perf / builtin-trace.c
index ab3ed4af1466433755cb5946ed10c23928f336b7..1a6cb7436952d0c5e75666967eba64c57fd8d34f 100644 (file)
+#include <traceevent/event-parse.h>
 #include "builtin.h"
 #include "util/color.h"
+#include "util/debug.h"
 #include "util/evlist.h"
 #include "util/machine.h"
+#include "util/session.h"
 #include "util/thread.h"
 #include "util/parse-options.h"
+#include "util/strlist.h"
+#include "util/intlist.h"
 #include "util/thread_map.h"
-#include "event-parse.h"
 
 #include <libaudit.h>
 #include <stdlib.h>
+#include <sys/mman.h>
+#include <linux/futex.h>
+
+static size_t syscall_arg__scnprintf_hex(char *bf, size_t size,
+                                        unsigned long arg, u8 *arg_mask __maybe_unused)
+{
+       return scnprintf(bf, size, "%#lx", arg);
+}
+
+#define SCA_HEX syscall_arg__scnprintf_hex
+
+static size_t syscall_arg__scnprintf_mmap_prot(char *bf, size_t size,
+                                              unsigned long arg, u8 *arg_mask __maybe_unused)
+{
+       int printed = 0, prot = arg;
+
+       if (prot == PROT_NONE)
+               return scnprintf(bf, size, "NONE");
+#define        P_MMAP_PROT(n) \
+       if (prot & PROT_##n) { \
+               printed += scnprintf(bf + printed, size - printed, "%s%s", printed ? "|" : "", #n); \
+               prot &= ~PROT_##n; \
+       }
+
+       P_MMAP_PROT(EXEC);
+       P_MMAP_PROT(READ);
+       P_MMAP_PROT(WRITE);
+#ifdef PROT_SEM
+       P_MMAP_PROT(SEM);
+#endif
+       P_MMAP_PROT(GROWSDOWN);
+       P_MMAP_PROT(GROWSUP);
+#undef P_MMAP_PROT
+
+       if (prot)
+               printed += scnprintf(bf + printed, size - printed, "%s%#x", printed ? "|" : "", prot);
+
+       return printed;
+}
+
+#define SCA_MMAP_PROT syscall_arg__scnprintf_mmap_prot
+
+static size_t syscall_arg__scnprintf_mmap_flags(char *bf, size_t size,
+                                               unsigned long arg, u8 *arg_mask __maybe_unused)
+{
+       int printed = 0, flags = arg;
+
+#define        P_MMAP_FLAG(n) \
+       if (flags & MAP_##n) { \
+               printed += scnprintf(bf + printed, size - printed, "%s%s", printed ? "|" : "", #n); \
+               flags &= ~MAP_##n; \
+       }
+
+       P_MMAP_FLAG(SHARED);
+       P_MMAP_FLAG(PRIVATE);
+       P_MMAP_FLAG(32BIT);
+       P_MMAP_FLAG(ANONYMOUS);
+       P_MMAP_FLAG(DENYWRITE);
+       P_MMAP_FLAG(EXECUTABLE);
+       P_MMAP_FLAG(FILE);
+       P_MMAP_FLAG(FIXED);
+       P_MMAP_FLAG(GROWSDOWN);
+#ifdef MAP_HUGETLB
+       P_MMAP_FLAG(HUGETLB);
+#endif
+       P_MMAP_FLAG(LOCKED);
+       P_MMAP_FLAG(NONBLOCK);
+       P_MMAP_FLAG(NORESERVE);
+       P_MMAP_FLAG(POPULATE);
+       P_MMAP_FLAG(STACK);
+#ifdef MAP_UNINITIALIZED
+       P_MMAP_FLAG(UNINITIALIZED);
+#endif
+#undef P_MMAP_FLAG
+
+       if (flags)
+               printed += scnprintf(bf + printed, size - printed, "%s%#x", printed ? "|" : "", flags);
+
+       return printed;
+}
+
+#define SCA_MMAP_FLAGS syscall_arg__scnprintf_mmap_flags
+
+static size_t syscall_arg__scnprintf_madvise_behavior(char *bf, size_t size,
+                                                     unsigned long arg, u8 *arg_mask __maybe_unused)
+{
+       int behavior = arg;
+
+       switch (behavior) {
+#define        P_MADV_BHV(n) case MADV_##n: return scnprintf(bf, size, #n)
+       P_MADV_BHV(NORMAL);
+       P_MADV_BHV(RANDOM);
+       P_MADV_BHV(SEQUENTIAL);
+       P_MADV_BHV(WILLNEED);
+       P_MADV_BHV(DONTNEED);
+       P_MADV_BHV(REMOVE);
+       P_MADV_BHV(DONTFORK);
+       P_MADV_BHV(DOFORK);
+       P_MADV_BHV(HWPOISON);
+#ifdef MADV_SOFT_OFFLINE
+       P_MADV_BHV(SOFT_OFFLINE);
+#endif
+       P_MADV_BHV(MERGEABLE);
+       P_MADV_BHV(UNMERGEABLE);
+#ifdef MADV_HUGEPAGE
+       P_MADV_BHV(HUGEPAGE);
+#endif
+#ifdef MADV_NOHUGEPAGE
+       P_MADV_BHV(NOHUGEPAGE);
+#endif
+#ifdef MADV_DONTDUMP
+       P_MADV_BHV(DONTDUMP);
+#endif
+#ifdef MADV_DODUMP
+       P_MADV_BHV(DODUMP);
+#endif
+#undef P_MADV_PHV
+       default: break;
+       }
+
+       return scnprintf(bf, size, "%#x", behavior);
+}
+
+#define SCA_MADV_BHV syscall_arg__scnprintf_madvise_behavior
+
+static size_t syscall_arg__scnprintf_futex_op(char *bf, size_t size, unsigned long arg, u8 *arg_mask)
+{
+       enum syscall_futex_args {
+               SCF_UADDR   = (1 << 0),
+               SCF_OP      = (1 << 1),
+               SCF_VAL     = (1 << 2),
+               SCF_TIMEOUT = (1 << 3),
+               SCF_UADDR2  = (1 << 4),
+               SCF_VAL3    = (1 << 5),
+       };
+       int op = arg;
+       int cmd = op & FUTEX_CMD_MASK;
+       size_t printed = 0;
+
+       switch (cmd) {
+#define        P_FUTEX_OP(n) case FUTEX_##n: printed = scnprintf(bf, size, #n);
+       P_FUTEX_OP(WAIT);           *arg_mask |= SCF_VAL3|SCF_UADDR2;             break;
+       P_FUTEX_OP(WAKE);           *arg_mask |= SCF_VAL3|SCF_UADDR2|SCF_TIMEOUT; break;
+       P_FUTEX_OP(FD);             *arg_mask |= SCF_VAL3|SCF_UADDR2|SCF_TIMEOUT; break;
+       P_FUTEX_OP(REQUEUE);        *arg_mask |= SCF_VAL3|SCF_TIMEOUT;            break;
+       P_FUTEX_OP(CMP_REQUEUE);    *arg_mask |= SCF_TIMEOUT;                     break;
+       P_FUTEX_OP(CMP_REQUEUE_PI); *arg_mask |= SCF_TIMEOUT;                     break;
+       P_FUTEX_OP(WAKE_OP);                                                      break;
+       P_FUTEX_OP(LOCK_PI);        *arg_mask |= SCF_VAL3|SCF_UADDR2|SCF_TIMEOUT; break;
+       P_FUTEX_OP(UNLOCK_PI);      *arg_mask |= SCF_VAL3|SCF_UADDR2|SCF_TIMEOUT; break;
+       P_FUTEX_OP(TRYLOCK_PI);     *arg_mask |= SCF_VAL3|SCF_UADDR2;             break;
+       P_FUTEX_OP(WAIT_BITSET);    *arg_mask |= SCF_UADDR2;                      break;
+       P_FUTEX_OP(WAKE_BITSET);    *arg_mask |= SCF_UADDR2;                      break;
+       P_FUTEX_OP(WAIT_REQUEUE_PI);                                              break;
+       default: printed = scnprintf(bf, size, "%#x", cmd);                       break;
+       }
+
+       if (op & FUTEX_PRIVATE_FLAG)
+               printed += scnprintf(bf + printed, size - printed, "|PRIV");
+
+       if (op & FUTEX_CLOCK_REALTIME)
+               printed += scnprintf(bf + printed, size - printed, "|CLKRT");
+
+       return printed;
+}
+
+#define SCA_FUTEX_OP  syscall_arg__scnprintf_futex_op
 
 static struct syscall_fmt {
        const char *name;
        const char *alias;
+       size_t     (*arg_scnprintf[6])(char *bf, size_t size, unsigned long arg, u8 *arg_mask);
        bool       errmsg;
        bool       timeout;
+       bool       hexret;
 } syscall_fmts[] = {
        { .name     = "access",     .errmsg = true, },
        { .name     = "arch_prctl", .errmsg = true, .alias = "prctl", },
+       { .name     = "brk",        .hexret = true,
+         .arg_scnprintf = { [0] = SCA_HEX, /* brk */ }, },
+       { .name     = "mmap",       .hexret = true, },
+       { .name     = "connect",    .errmsg = true, },
        { .name     = "fstat",      .errmsg = true, .alias = "newfstat", },
        { .name     = "fstatat",    .errmsg = true, .alias = "newfstatat", },
-       { .name     = "futex",      .errmsg = true, },
+       { .name     = "futex",      .errmsg = true,
+         .arg_scnprintf = { [1] = SCA_FUTEX_OP, /* op */ }, },
+       { .name     = "ioctl",      .errmsg = true,
+         .arg_scnprintf = { [2] = SCA_HEX, /* arg */ }, },
+       { .name     = "lstat",      .errmsg = true, .alias = "newlstat", },
+       { .name     = "madvise",    .errmsg = true,
+         .arg_scnprintf = { [0] = SCA_HEX,      /* start */
+                            [2] = SCA_MADV_BHV, /* behavior */ }, },
+       { .name     = "mmap",       .hexret = true,
+         .arg_scnprintf = { [0] = SCA_HEX,       /* addr */
+                            [2] = SCA_MMAP_PROT, /* prot */
+                            [3] = SCA_MMAP_FLAGS, /* flags */ }, },
+       { .name     = "mprotect",   .errmsg = true,
+         .arg_scnprintf = { [0] = SCA_HEX, /* start */
+                            [2] = SCA_MMAP_PROT, /* prot */ }, },
+       { .name     = "mremap",     .hexret = true,
+         .arg_scnprintf = { [0] = SCA_HEX, /* addr */
+                            [4] = SCA_HEX, /* new_addr */ }, },
+       { .name     = "munmap",     .errmsg = true,
+         .arg_scnprintf = { [0] = SCA_HEX, /* addr */ }, },
        { .name     = "open",       .errmsg = true, },
        { .name     = "poll",       .errmsg = true, .timeout = true, },
        { .name     = "ppoll",      .errmsg = true, .timeout = true, },
+       { .name     = "pread",      .errmsg = true, .alias = "pread64", },
+       { .name     = "pwrite",     .errmsg = true, .alias = "pwrite64", },
        { .name     = "read",       .errmsg = true, },
        { .name     = "recvfrom",   .errmsg = true, },
        { .name     = "select",     .errmsg = true, .timeout = true, },
        { .name     = "socket",     .errmsg = true, },
        { .name     = "stat",       .errmsg = true, .alias = "newstat", },
+       { .name     = "uname",      .errmsg = true, .alias = "newuname", },
 };
 
 static int syscall_fmt__cmp(const void *name, const void *fmtp)
@@ -46,7 +245,10 @@ static struct syscall_fmt *syscall_fmt__find(const char *name)
 struct syscall {
        struct event_format *tp_format;
        const char          *name;
+       bool                filtered;
        struct syscall_fmt  *fmt;
+       size_t              (**arg_scnprintf)(char *bf, size_t size,
+                                             unsigned long arg, u8 *args_mask);
 };
 
 static size_t fprintf_duration(unsigned long t, FILE *fp)
@@ -60,7 +262,7 @@ static size_t fprintf_duration(unsigned long t, FILE *fp)
                printed += color_fprintf(fp, PERF_COLOR_YELLOW, "%6.3f ms", duration);
        else
                printed += color_fprintf(fp, PERF_COLOR_NORMAL, "%6.3f ms", duration);
-       return printed + fprintf(stdout, "): ");
+       return printed + fprintf(fp, "): ");
 }
 
 struct thread_trace {
@@ -77,7 +279,7 @@ static struct thread_trace *thread_trace__new(void)
        return zalloc(sizeof(struct thread_trace));
 }
 
-static struct thread_trace *thread__trace(struct thread *thread)
+static struct thread_trace *thread__trace(struct thread *thread, FILE *fp)
 {
        struct thread_trace *ttrace;
 
@@ -95,12 +297,13 @@ static struct thread_trace *thread__trace(struct thread *thread)
 
        return ttrace;
 fail:
-       color_fprintf(stdout, PERF_COLOR_RED,
+       color_fprintf(fp, PERF_COLOR_RED,
                      "WARNING: not enough memory, dropping samples!\n");
        return NULL;
 }
 
 struct trace {
+       struct perf_tool        tool;
        int                     audit_machine;
        struct {
                int             max;
@@ -109,7 +312,12 @@ struct trace {
        struct perf_record_opts opts;
        struct machine          host;
        u64                     base_time;
+       FILE                    *output;
        unsigned long           nr_events;
+       struct strlist          *ev_qualifier;
+       bool                    not_ev_qualifier;
+       struct intlist          *tid_list;
+       struct intlist          *pid_list;
        bool                    sched;
        bool                    multiple_threads;
        double                  duration_filter;
@@ -142,18 +350,19 @@ static size_t trace__fprintf_entry_head(struct trace *trace, struct thread *thre
        printed += fprintf_duration(duration, fp);
 
        if (trace->multiple_threads)
-               printed += fprintf(fp, "%d ", thread->pid);
+               printed += fprintf(fp, "%d ", thread->tid);
 
        return printed;
 }
 
-static int trace__process_event(struct machine *machine, union perf_event *event)
+static int trace__process_event(struct trace *trace, struct machine *machine,
+                               union perf_event *event)
 {
        int ret = 0;
 
        switch (event->header.type) {
        case PERF_RECORD_LOST:
-               color_fprintf(stdout, PERF_COLOR_RED,
+               color_fprintf(trace->output, PERF_COLOR_RED,
                              "LOST %" PRIu64 " events!\n", event->lost.lost);
                ret = machine__process_lost_event(machine, event);
        default:
@@ -164,12 +373,13 @@ static int trace__process_event(struct machine *machine, union perf_event *event
        return ret;
 }
 
-static int trace__tool_process(struct perf_tool *tool __maybe_unused,
+static int trace__tool_process(struct perf_tool *tool,
                               union perf_event *event,
                               struct perf_sample *sample __maybe_unused,
                               struct machine *machine)
 {
-       return trace__process_event(machine, event);
+       struct trace *trace = container_of(tool, struct trace, tool);
+       return trace__process_event(trace, machine, event);
 }
 
 static int trace__symbols_init(struct trace *trace, struct perf_evlist *evlist)
@@ -183,11 +393,11 @@ static int trace__symbols_init(struct trace *trace, struct perf_evlist *evlist)
        machine__create_kernel_maps(&trace->host);
 
        if (perf_target__has_task(&trace->opts.target)) {
-               err = perf_event__synthesize_thread_map(NULL, evlist->threads,
+               err = perf_event__synthesize_thread_map(&trace->tool, evlist->threads,
                                                        trace__tool_process,
                                                        &trace->host);
        } else {
-               err = perf_event__synthesize_threads(NULL, trace__tool_process,
+               err = perf_event__synthesize_threads(&trace->tool, trace__tool_process,
                                                     &trace->host);
        }
 
@@ -197,6 +407,26 @@ static int trace__symbols_init(struct trace *trace, struct perf_evlist *evlist)
        return err;
 }
 
+static int syscall__set_arg_fmts(struct syscall *sc)
+{
+       struct format_field *field;
+       int idx = 0;
+
+       sc->arg_scnprintf = calloc(sc->tp_format->format.nr_fields - 1, sizeof(void *));
+       if (sc->arg_scnprintf == NULL)
+               return -1;
+
+       for (field = sc->tp_format->format.fields->next; field; field = field->next) {
+               if (sc->fmt && sc->fmt->arg_scnprintf[idx])
+                       sc->arg_scnprintf[idx] = sc->fmt->arg_scnprintf[idx];
+               else if (field->flags & FIELD_IS_POINTER)
+                       sc->arg_scnprintf[idx] = syscall_arg__scnprintf_hex;
+               ++idx;
+       }
+
+       return 0;
+}
+
 static int trace__read_syscall_info(struct trace *trace, int id)
 {
        char tp_name[128];
@@ -225,6 +455,20 @@ static int trace__read_syscall_info(struct trace *trace, int id)
 
        sc = trace->syscalls.table + id;
        sc->name = name;
+
+       if (trace->ev_qualifier) {
+               bool in = strlist__find(trace->ev_qualifier, name) != NULL;
+
+               if (!(in ^ trace->not_ev_qualifier)) {
+                       sc->filtered = true;
+                       /*
+                        * No need to do read tracepoint information since this will be
+                        * filtered out.
+                        */
+                       return 0;
+               }
+       }
+
        sc->fmt  = syscall_fmt__find(sc->name);
 
        snprintf(tp_name, sizeof(tp_name), "sys_enter_%s", sc->name);
@@ -235,7 +479,10 @@ static int trace__read_syscall_info(struct trace *trace, int id)
                sc->tp_format = event_format__new("syscalls", tp_name);
        }
 
-       return sc->tp_format != NULL ? 0 : -1;
+       if (sc->tp_format == NULL)
+               return -1;
+
+       return syscall__set_arg_fmts(sc);
 }
 
 static size_t syscall__scnprintf_args(struct syscall *sc, char *bf, size_t size,
@@ -246,11 +493,23 @@ static size_t syscall__scnprintf_args(struct syscall *sc, char *bf, size_t size,
 
        if (sc->tp_format != NULL) {
                struct format_field *field;
+               u8 mask = 0, bit = 1;
+
+               for (field = sc->tp_format->format.fields->next; field;
+                    field = field->next, ++i, bit <<= 1) {
+                       if (mask & bit)
+                               continue;
 
-               for (field = sc->tp_format->format.fields->next; field; field = field->next) {
                        printed += scnprintf(bf + printed, size - printed,
-                                            "%s%s: %ld", printed ? ", " : "",
-                                            field->name, args[i++]);
+                                            "%s%s: ", printed ? ", " : "", field->name);
+
+                       if (sc->arg_scnprintf && sc->arg_scnprintf[i]) {
+                               printed += sc->arg_scnprintf[i](bf + printed, size - printed,
+                                                               args[i], &mask);
+                       } else {
+                               printed += scnprintf(bf + printed, size - printed,
+                                                    "%ld", args[i]);
+                       }
                }
        } else {
                while (i < 6) {
@@ -274,7 +533,22 @@ static struct syscall *trace__syscall_info(struct trace *trace,
        int id = perf_evsel__intval(evsel, sample, "id");
 
        if (id < 0) {
-               printf("Invalid syscall %d id, skipping...\n", id);
+
+               /*
+                * XXX: Noticed on x86_64, reproduced as far back as 3.0.36, haven't tried
+                * before that, leaving at a higher verbosity level till that is
+                * explained. Reproduced with plain ftrace with:
+                *
+                * echo 1 > /t/events/raw_syscalls/sys_exit/enable
+                * grep "NR -1 " /t/trace_pipe
+                *
+                * After generating some load on the machine.
+                */
+               if (verbose > 1) {
+                       static u64 n;
+                       fprintf(trace->output, "Invalid syscall %d id, skipping (%s, %" PRIu64 ") ...\n",
+                               id, perf_evsel__name(evsel), ++n);
+               }
                return NULL;
        }
 
@@ -288,10 +562,12 @@ static struct syscall *trace__syscall_info(struct trace *trace,
        return &trace->syscalls.table[id];
 
 out_cant_read:
-       printf("Problems reading syscall %d", id);
-       if (id <= trace->syscalls.max && trace->syscalls.table[id].name != NULL)
-               printf("(%s)", trace->syscalls.table[id].name);
-       puts(" information");
+       if (verbose) {
+               fprintf(trace->output, "Problems reading syscall %d", id);
+               if (id <= trace->syscalls.max && trace->syscalls.table[id].name != NULL)
+                       fprintf(trace->output, "(%s)", trace->syscalls.table[id].name);
+               fputs(" information\n", trace->output);
+       }
        return NULL;
 }
 
@@ -301,16 +577,25 @@ static int trace__sys_enter(struct trace *trace, struct perf_evsel *evsel,
        char *msg;
        void *args;
        size_t printed = 0;
-       struct thread *thread = machine__findnew_thread(&trace->host, sample->tid);
+       struct thread *thread;
        struct syscall *sc = trace__syscall_info(trace, evsel, sample);
-       struct thread_trace *ttrace = thread__trace(thread);
+       struct thread_trace *ttrace;
+
+       if (sc == NULL)
+               return -1;
 
-       if (ttrace == NULL || sc == NULL)
+       if (sc->filtered)
+               return 0;
+
+       thread = machine__findnew_thread(&trace->host, sample->pid,
+                                        sample->tid);
+       ttrace = thread__trace(thread, trace->output);
+       if (ttrace == NULL)
                return -1;
 
        args = perf_evsel__rawptr(evsel, sample, "args");
        if (args == NULL) {
-               printf("Problems reading syscall arguments\n");
+               fprintf(trace->output, "Problems reading syscall arguments\n");
                return -1;
        }
 
@@ -330,8 +615,8 @@ static int trace__sys_enter(struct trace *trace, struct perf_evsel *evsel,
 
        if (!strcmp(sc->name, "exit_group") || !strcmp(sc->name, "exit")) {
                if (!trace->duration_filter) {
-                       trace__fprintf_entry_head(trace, thread, 1, sample->time, stdout);
-                       printf("%-70s\n", ttrace->entry_str);
+                       trace__fprintf_entry_head(trace, thread, 1, sample->time, trace->output);
+                       fprintf(trace->output, "%-70s\n", ttrace->entry_str);
                }
        } else
                ttrace->entry_pending = true;
@@ -344,11 +629,20 @@ static int trace__sys_exit(struct trace *trace, struct perf_evsel *evsel,
 {
        int ret;
        u64 duration = 0;
-       struct thread *thread = machine__findnew_thread(&trace->host, sample->tid);
-       struct thread_trace *ttrace = thread__trace(thread);
+       struct thread *thread;
        struct syscall *sc = trace__syscall_info(trace, evsel, sample);
+       struct thread_trace *ttrace;
+
+       if (sc == NULL)
+               return -1;
 
-       if (ttrace == NULL || sc == NULL)
+       if (sc->filtered)
+               return 0;
+
+       thread = machine__findnew_thread(&trace->host, sample->pid,
+                                        sample->tid);
+       ttrace = thread__trace(thread, trace->output);
+       if (ttrace == NULL)
                return -1;
 
        ret = perf_evsel__intval(evsel, sample, "ret");
@@ -364,28 +658,33 @@ static int trace__sys_exit(struct trace *trace, struct perf_evsel *evsel,
        } else if (trace->duration_filter)
                goto out;
 
-       trace__fprintf_entry_head(trace, thread, duration, sample->time, stdout);
+       trace__fprintf_entry_head(trace, thread, duration, sample->time, trace->output);
 
        if (ttrace->entry_pending) {
-               printf("%-70s", ttrace->entry_str);
+               fprintf(trace->output, "%-70s", ttrace->entry_str);
        } else {
-               printf(" ... [");
-               color_fprintf(stdout, PERF_COLOR_YELLOW, "continued");
-               printf("]: %s()", sc->name);
+               fprintf(trace->output, " ... [");
+               color_fprintf(trace->output, PERF_COLOR_YELLOW, "continued");
+               fprintf(trace->output, "]: %s()", sc->name);
        }
 
-       if (ret < 0 && sc->fmt && sc->fmt->errmsg) {
+       if (sc->fmt == NULL) {
+signed_print:
+               fprintf(trace->output, ") = %d", ret);
+       } else if (ret < 0 && sc->fmt->errmsg) {
                char bf[256];
                const char *emsg = strerror_r(-ret, bf, sizeof(bf)),
                           *e = audit_errno_to_name(-ret);
 
-               printf(") = -1 %s %s", e, emsg);
-       } else if (ret == 0 && sc->fmt && sc->fmt->timeout)
-               printf(") = 0 Timeout");
+               fprintf(trace->output, ") = -1 %s %s", e, emsg);
+       } else if (ret == 0 && sc->fmt->timeout)
+               fprintf(trace->output, ") = 0 Timeout");
+       else if (sc->fmt->hexret)
+               fprintf(trace->output, ") = %#x", ret);
        else
-               printf(") = %d", ret);
+               goto signed_print;
 
-       putchar('\n');
+       fputc('\n', trace->output);
 out:
        ttrace->entry_pending = false;
 
@@ -397,8 +696,10 @@ static int trace__sched_stat_runtime(struct trace *trace, struct perf_evsel *evs
 {
         u64 runtime = perf_evsel__intval(evsel, sample, "runtime");
        double runtime_ms = (double)runtime / NSEC_PER_MSEC;
-       struct thread *thread = machine__findnew_thread(&trace->host, sample->tid);
-       struct thread_trace *ttrace = thread__trace(thread);
+       struct thread *thread = machine__findnew_thread(&trace->host,
+                                                       sample->pid,
+                                                       sample->tid);
+       struct thread_trace *ttrace = thread__trace(thread, trace->output);
 
        if (ttrace == NULL)
                goto out_dump;
@@ -408,7 +709,7 @@ static int trace__sched_stat_runtime(struct trace *trace, struct perf_evsel *evs
        return 0;
 
 out_dump:
-       printf("%s: comm=%s,pid=%u,runtime=%" PRIu64 ",vruntime=%" PRIu64 ")\n",
+       fprintf(trace->output, "%s: comm=%s,pid=%u,runtime=%" PRIu64 ",vruntime=%" PRIu64 ")\n",
               evsel->name,
               perf_evsel__strval(evsel, sample, "comm"),
               (pid_t)perf_evsel__intval(evsel, sample, "pid"),
@@ -417,6 +718,72 @@ out_dump:
        return 0;
 }
 
+static bool skip_sample(struct trace *trace, struct perf_sample *sample)
+{
+       if ((trace->pid_list && intlist__find(trace->pid_list, sample->pid)) ||
+           (trace->tid_list && intlist__find(trace->tid_list, sample->tid)))
+               return false;
+
+       if (trace->pid_list || trace->tid_list)
+               return true;
+
+       return false;
+}
+
+static int trace__process_sample(struct perf_tool *tool,
+                                union perf_event *event __maybe_unused,
+                                struct perf_sample *sample,
+                                struct perf_evsel *evsel,
+                                struct machine *machine __maybe_unused)
+{
+       struct trace *trace = container_of(tool, struct trace, tool);
+       int err = 0;
+
+       tracepoint_handler handler = evsel->handler.func;
+
+       if (skip_sample(trace, sample))
+               return 0;
+
+       if (trace->base_time == 0)
+               trace->base_time = sample->time;
+
+       if (handler)
+               handler(trace, evsel, sample);
+
+       return err;
+}
+
+static bool
+perf_session__has_tp(struct perf_session *session, const char *name)
+{
+       struct perf_evsel *evsel;
+
+       evsel = perf_evlist__find_tracepoint_by_name(session->evlist, name);
+
+       return evsel != NULL;
+}
+
+static int parse_target_str(struct trace *trace)
+{
+       if (trace->opts.target.pid) {
+               trace->pid_list = intlist__new(trace->opts.target.pid);
+               if (trace->pid_list == NULL) {
+                       pr_err("Error parsing process id string\n");
+                       return -EINVAL;
+               }
+       }
+
+       if (trace->opts.target.tid) {
+               trace->tid_list = intlist__new(trace->opts.target.tid);
+               if (trace->tid_list == NULL) {
+                       pr_err("Error parsing thread id string\n");
+                       return -EINVAL;
+               }
+       }
+
+       return 0;
+}
+
 static int trace__run(struct trace *trace, int argc, const char **argv)
 {
        struct perf_evlist *evlist = perf_evlist__new();
@@ -426,32 +793,32 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
        const bool forks = argc > 0;
 
        if (evlist == NULL) {
-               printf("Not enough memory to run!\n");
+               fprintf(trace->output, "Not enough memory to run!\n");
                goto out;
        }
 
        if (perf_evlist__add_newtp(evlist, "raw_syscalls", "sys_enter", trace__sys_enter) ||
            perf_evlist__add_newtp(evlist, "raw_syscalls", "sys_exit", trace__sys_exit)) {
-               printf("Couldn't read the raw_syscalls tracepoints information!\n");
+               fprintf(trace->output, "Couldn't read the raw_syscalls tracepoints information!\n");
                goto out_delete_evlist;
        }
 
        if (trace->sched &&
            perf_evlist__add_newtp(evlist, "sched", "sched_stat_runtime",
                                   trace__sched_stat_runtime)) {
-               printf("Couldn't read the sched_stat_runtime tracepoint information!\n");
+               fprintf(trace->output, "Couldn't read the sched_stat_runtime tracepoint information!\n");
                goto out_delete_evlist;
        }
 
        err = perf_evlist__create_maps(evlist, &trace->opts.target);
        if (err < 0) {
-               printf("Problems parsing the target to trace, check your options!\n");
+               fprintf(trace->output, "Problems parsing the target to trace, check your options!\n");
                goto out_delete_evlist;
        }
 
        err = trace__symbols_init(trace, evlist);
        if (err < 0) {
-               printf("Problems initializing symbol libraries!\n");
+               fprintf(trace->output, "Problems initializing symbol libraries!\n");
                goto out_delete_maps;
        }
 
@@ -464,20 +831,20 @@ static int trace__run(struct trace *trace, int argc, const char **argv)
                err = perf_evlist__prepare_workload(evlist, &trace->opts.target,
                                                    argv, false, false);
                if (err < 0) {
-                       printf("Couldn't run the workload!\n");
+                       fprintf(trace->output, "Couldn't run the workload!\n");
                        goto out_delete_maps;
                }
        }
 
        err = perf_evlist__open(evlist);
        if (err < 0) {
-               printf("Couldn't create the events: %s\n", strerror(errno));
+               fprintf(trace->output, "Couldn't create the events: %s\n", strerror(errno));
                goto out_delete_maps;
        }
 
        err = perf_evlist__mmap(evlist, UINT_MAX, false);
        if (err < 0) {
-               printf("Couldn't mmap the events: %s\n", strerror(errno));
+               fprintf(trace->output, "Couldn't mmap the events: %s\n", strerror(errno));
                goto out_close_evlist;
        }
 
@@ -502,7 +869,7 @@ again:
 
                        err = perf_evlist__parse_sample(evlist, event, &sample);
                        if (err) {
-                               printf("Can't parse sample, err = %d, skipping...\n", err);
+                               fprintf(trace->output, "Can't parse sample, err = %d, skipping...\n", err);
                                continue;
                        }
 
@@ -510,18 +877,18 @@ again:
                                trace->base_time = sample.time;
 
                        if (type != PERF_RECORD_SAMPLE) {
-                               trace__process_event(&trace->host, event);
+                               trace__process_event(trace, &trace->host, event);
                                continue;
                        }
 
                        evsel = perf_evlist__id2evsel(evlist, sample.id);
                        if (evsel == NULL) {
-                               printf("Unknown tp ID %" PRIu64 ", skipping...\n", sample.id);
+                               fprintf(trace->output, "Unknown tp ID %" PRIu64 ", skipping...\n", sample.id);
                                continue;
                        }
 
                        if (sample.raw_data == NULL) {
-                               printf("%s sample with no payload for tid: %d, cpu %d, raw_size=%d, skipping...\n",
+                               fprintf(trace->output, "%s sample with no payload for tid: %d, cpu %d, raw_size=%d, skipping...\n",
                                       perf_evsel__name(evsel), sample.tid,
                                       sample.cpu, sample.raw_size);
                                continue;
@@ -556,6 +923,69 @@ out:
        return err;
 }
 
+static int trace__replay(struct trace *trace)
+{
+       const struct perf_evsel_str_handler handlers[] = {
+               { "raw_syscalls:sys_enter",  trace__sys_enter, },
+               { "raw_syscalls:sys_exit",   trace__sys_exit, },
+       };
+
+       struct perf_session *session;
+       int err = -1;
+
+       trace->tool.sample        = trace__process_sample;
+       trace->tool.mmap          = perf_event__process_mmap;
+       trace->tool.comm          = perf_event__process_comm;
+       trace->tool.exit          = perf_event__process_exit;
+       trace->tool.fork          = perf_event__process_fork;
+       trace->tool.attr          = perf_event__process_attr;
+       trace->tool.tracing_data = perf_event__process_tracing_data;
+       trace->tool.build_id      = perf_event__process_build_id;
+
+       trace->tool.ordered_samples = true;
+       trace->tool.ordering_requires_timestamps = true;
+
+       /* add tid to output */
+       trace->multiple_threads = true;
+
+       if (symbol__init() < 0)
+               return -1;
+
+       session = perf_session__new(input_name, O_RDONLY, 0, false,
+                                   &trace->tool);
+       if (session == NULL)
+               return -ENOMEM;
+
+       err = perf_session__set_tracepoints_handlers(session, handlers);
+       if (err)
+               goto out;
+
+       if (!perf_session__has_tp(session, "raw_syscalls:sys_enter")) {
+               pr_err("Data file does not have raw_syscalls:sys_enter events\n");
+               goto out;
+       }
+
+       if (!perf_session__has_tp(session, "raw_syscalls:sys_exit")) {
+               pr_err("Data file does not have raw_syscalls:sys_exit events\n");
+               goto out;
+       }
+
+       err = parse_target_str(trace);
+       if (err != 0)
+               goto out;
+
+       setup_pager();
+
+       err = perf_session__process_events(session, &trace->tool);
+       if (err)
+               pr_err("Failed to process events, error %d", err);
+
+out:
+       perf_session__delete(session);
+
+       return err;
+}
+
 static size_t trace__fprintf_threads_header(FILE *fp)
 {
        size_t printed;
@@ -593,7 +1023,7 @@ static size_t trace__fprintf_thread_summary(struct trace *trace, FILE *fp)
                        color = PERF_COLOR_YELLOW;
 
                printed += color_fprintf(fp, color, "%20s", thread->comm);
-               printed += fprintf(fp, " - %-5d :%11lu   [", thread->pid, ttrace->nr_events);
+               printed += fprintf(fp, " - %-5d :%11lu   [", thread->tid, ttrace->nr_events);
                printed += color_fprintf(fp, color, "%5.1f%%", ratio);
                printed += fprintf(fp, " ] %10.3f ms\n", ttrace->runtime_ms);
        }
@@ -610,6 +1040,23 @@ static int trace__set_duration(const struct option *opt, const char *str,
        return 0;
 }
 
+static int trace__open_output(struct trace *trace, const char *filename)
+{
+       struct stat st;
+
+       if (!stat(filename, &st) && st.st_size) {
+               char oldname[PATH_MAX];
+
+               scnprintf(oldname, sizeof(oldname), "%s.old", filename);
+               unlink(oldname);
+               rename(filename, oldname);
+       }
+
+       trace->output = fopen(filename, "w");
+
+       return trace->output == NULL ? -errno : 0;
+}
+
 int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused)
 {
        const char * const trace_usage[] = {
@@ -632,26 +1079,34 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused)
                        .no_delay      = true,
                        .mmap_pages    = 1024,
                },
+               .output = stdout,
        };
+       const char *output_name = NULL;
+       const char *ev_qualifier_str = NULL;
        const struct option trace_options[] = {
+       OPT_STRING('e', "expr", &ev_qualifier_str, "expr",
+                   "list of events to trace"),
+       OPT_STRING('o', "output", &output_name, "file", "output file name"),
+       OPT_STRING('i', "input", &input_name, "file", "Analyze events in file"),
        OPT_STRING('p', "pid", &trace.opts.target.pid, "pid",
                    "trace events on existing process id"),
-       OPT_STRING(0, "tid", &trace.opts.target.tid, "tid",
+       OPT_STRING('t', "tid", &trace.opts.target.tid, "tid",
                    "trace events on existing thread id"),
-       OPT_BOOLEAN(0, "all-cpus", &trace.opts.target.system_wide,
+       OPT_BOOLEAN('a', "all-cpus", &trace.opts.target.system_wide,
                    "system-wide collection from all CPUs"),
-       OPT_STRING(0, "cpu", &trace.opts.target.cpu_list, "cpu",
+       OPT_STRING('C', "cpu", &trace.opts.target.cpu_list, "cpu",
                    "list of cpus to monitor"),
        OPT_BOOLEAN(0, "no-inherit", &trace.opts.no_inherit,
                    "child tasks do not inherit counters"),
-       OPT_UINTEGER(0, "mmap-pages", &trace.opts.mmap_pages,
+       OPT_UINTEGER('m', "mmap-pages", &trace.opts.mmap_pages,
                     "number of mmap data pages"),
-       OPT_STRING(0, "uid", &trace.opts.target.uid_str, "user",
+       OPT_STRING('u', "uid", &trace.opts.target.uid_str, "user",
                   "user to profile"),
        OPT_CALLBACK(0, "duration", &trace, "float",
                     "show only events with duration > N.M ms",
                     trace__set_duration),
        OPT_BOOLEAN(0, "sched", &trace.sched, "show blocking scheduler events"),
+       OPT_INCR('v', "verbose", &verbose, "be more verbose"),
        OPT_END()
        };
        int err;
@@ -659,27 +1114,57 @@ int cmd_trace(int argc, const char **argv, const char *prefix __maybe_unused)
 
        argc = parse_options(argc, argv, trace_options, trace_usage, 0);
 
+       if (output_name != NULL) {
+               err = trace__open_output(&trace, output_name);
+               if (err < 0) {
+                       perror("failed to create output file");
+                       goto out;
+               }
+       }
+
+       if (ev_qualifier_str != NULL) {
+               const char *s = ev_qualifier_str;
+
+               trace.not_ev_qualifier = *s == '!';
+               if (trace.not_ev_qualifier)
+                       ++s;
+               trace.ev_qualifier = strlist__new(true, s);
+               if (trace.ev_qualifier == NULL) {
+                       fputs("Not enough memory to parse event qualifier",
+                             trace.output);
+                       err = -ENOMEM;
+                       goto out_close;
+               }
+       }
+
        err = perf_target__validate(&trace.opts.target);
        if (err) {
                perf_target__strerror(&trace.opts.target, err, bf, sizeof(bf));
-               printf("%s", bf);
-               return err;
+               fprintf(trace.output, "%s", bf);
+               goto out_close;
        }
 
        err = perf_target__parse_uid(&trace.opts.target);
        if (err) {
                perf_target__strerror(&trace.opts.target, err, bf, sizeof(bf));
-               printf("%s", bf);
-               return err;
+               fprintf(trace.output, "%s", bf);
+               goto out_close;
        }
 
        if (!argc && perf_target__none(&trace.opts.target))
                trace.opts.target.system_wide = true;
 
-       err = trace__run(&trace, argc, argv);
+       if (input_name)
+               err = trace__replay(&trace);
+       else
+               err = trace__run(&trace, argc, argv);
 
        if (trace.sched && !err)
-               trace__fprintf_thread_summary(&trace, stdout);
+               trace__fprintf_thread_summary(&trace, trace.output);
 
+out_close:
+       if (output_name != NULL)
+               fclose(trace.output);
+out:
        return err;
 }