kernel/trace/trace.c

   1 /*
   2  * ring buffer based function tracer
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally taken from the RT patch by:
   8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code from the latency_tracer, that is:
  11  *  Copyright (C) 2004-2006 Ingo Molnar
  12  *  Copyright (C) 2004 William Lee Irwin III
  13  */
  14 #include <linux/utsrelease.h>
  15 #include <linux/kallsyms.h>
  16 #include <linux/seq_file.h>
  17 #include <linux/debugfs.h>
  18 #include <linux/pagemap.h>
  19 #include <linux/hardirq.h>
  20 #include <linux/linkage.h>
  21 #include <linux/uaccess.h>
  22 #include <linux/ftrace.h>
  23 #include <linux/module.h>
  24 #include <linux/percpu.h>
  25 #include <linux/ctype.h>
  26 #include <linux/init.h>
  27 #include <linux/gfp.h>
  28 #include <linux/fs.h>
  29
  30 #include "trace.h"
  31
  32 unsigned long __read_mostly     tracing_max_latency = (cycle_t)ULONG_MAX;
  33 unsigned long __read_mostly     tracing_thresh;
  34
  35 static int tracing_disabled = 1;
  36
  37 static long notrace
  38 ns2usecs(cycle_t nsec)
  39 {
  40         nsec += 500;
  41         do_div(nsec, 1000);
  42         return nsec;
  43 }
  44
  45 notrace cycle_t ftrace_now(int cpu)
  46 {
  47         return cpu_clock(cpu);
  48 }
  49
  50 static struct trace_array       global_trace;
  51
  52 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
  53
  54 static struct trace_array       max_tr;
  55
  56 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
  57
  58 static int                      tracer_enabled = 1;
  59 static unsigned long            trace_nr_entries = 16384UL;
  60
  61 static struct tracer            *trace_types __read_mostly;
  62 static struct tracer            *current_trace __read_mostly;
  63 static int                      max_tracer_type_len;
  64
  65 static DEFINE_MUTEX(trace_types_lock);
  66
  67 #define ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(struct trace_entry))
  68
  69 static int __init set_nr_entries(char *str)
  70 {
  71         if (!str)
  72                 return 0;
  73         trace_nr_entries = simple_strtoul(str, &str, 0);
  74         return 1;
  75 }
  76 __setup("trace_entries=", set_nr_entries);
  77
  78 unsigned long nsecs_to_usecs(unsigned long nsecs)
  79 {
  80         return nsecs / 1000;
  81 }
  82
  83 enum trace_type {
  84         __TRACE_FIRST_TYPE = 0,
  85
  86         TRACE_FN,
  87         TRACE_CTX,
  88         TRACE_SPECIAL,
  89
  90         __TRACE_LAST_TYPE
  91 };
  92
  93 enum trace_flag_type {
  94         TRACE_FLAG_IRQS_OFF             = 0x01,
  95         TRACE_FLAG_NEED_RESCHED         = 0x02,
  96         TRACE_FLAG_HARDIRQ              = 0x04,
  97         TRACE_FLAG_SOFTIRQ              = 0x08,
  98 };
  99
 100 enum trace_iterator_flags {
 101         TRACE_ITER_PRINT_PARENT         = 0x01,
 102         TRACE_ITER_SYM_OFFSET           = 0x02,
 103         TRACE_ITER_SYM_ADDR             = 0x04,
 104         TRACE_ITER_VERBOSE              = 0x08,
 105         TRACE_ITER_RAW                  = 0x10,
 106         TRACE_ITER_BIN                  = 0x20,
 107 };
 108
 109 #define TRACE_ITER_SYM_MASK \
 110         (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
 111
 112 /* These must match the bit postions above */
 113 static const char *trace_options[] = {
 114         "print-parent",
 115         "sym-offset",
 116         "sym-addr",
 117         "verbose",
 118         "raw",
 119         "bin",
 120         NULL
 121 };
 122
 123 static unsigned trace_flags;
 124
 125 static DEFINE_SPINLOCK(ftrace_max_lock);
 126
 127 /*
 128  * Copy the new maximum trace into the separate maximum-trace
 129  * structure. (this way the maximum trace is permanently saved,
 130  * for later retrieval via /debugfs/tracing/latency_trace)
 131  */
 132 static notrace void
 133 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 134 {
 135         struct trace_array_cpu *data = tr->data[cpu];
 136
 137         max_tr.cpu = cpu;
 138         max_tr.time_start = data->preempt_timestamp;
 139
 140         data = max_tr.data[cpu];
 141         data->saved_latency = tracing_max_latency;
 142
 143         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
 144         data->pid = tsk->pid;
 145         data->uid = tsk->uid;
 146         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
 147         data->policy = tsk->policy;
 148         data->rt_priority = tsk->rt_priority;
 149
 150         /* record this tasks comm */
 151         tracing_record_cmdline(current);
 152 }
 153
 154 void check_pages(struct trace_array_cpu *data)
 155 {
 156         struct page *page, *tmp;
 157
 158         BUG_ON(data->trace_pages.next->prev != &data->trace_pages);
 159         BUG_ON(data->trace_pages.prev->next != &data->trace_pages);
 160
 161         list_for_each_entry_safe(page, tmp, &data->trace_pages, lru) {
 162                 BUG_ON(page->lru.next->prev != &page->lru);
 163                 BUG_ON(page->lru.prev->next != &page->lru);
 164         }
 165 }
 166
 167 void *head_page(struct trace_array_cpu *data)
 168 {
 169         struct page *page;
 170
 171         check_pages(data);
 172         if (list_empty(&data->trace_pages))
 173                 return NULL;
 174
 175         page = list_entry(data->trace_pages.next, struct page, lru);
 176         BUG_ON(&page->lru == &data->trace_pages);
 177
 178         return page_address(page);
 179 }
 180
 181 static notrace int
 182 trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
 183 {
 184         int len = (PAGE_SIZE - 1) - s->len;
 185         va_list ap;
 186         int ret;
 187
 188         if (!len)
 189                 return 0;
 190
 191         va_start(ap, fmt);
 192         ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
 193         va_end(ap);
 194
 195         /* If we can't write it all, don't bother writing anything */
 196         if (ret > len)
 197                 return 0;
 198
 199         s->len += ret;
 200
 201         return len;
 202 }
 203
 204 static notrace int
 205 trace_seq_puts(struct trace_seq *s, const char *str)
 206 {
 207         int len = strlen(str);
 208
 209         if (len > ((PAGE_SIZE - 1) - s->len))
 210                 return 0;
 211
 212         memcpy(s->buffer + s->len, str, len);
 213         s->len += len;
 214
 215         return len;
 216 }
 217
 218 static notrace int
 219 trace_seq_putc(struct trace_seq *s, unsigned char c)
 220 {
 221         if (s->len >= (PAGE_SIZE - 1))
 222                 return 0;
 223
 224         s->buffer[s->len++] = c;
 225
 226         return 1;
 227 }
 228
 229 static notrace int
 230 trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
 231 {
 232         if (len > ((PAGE_SIZE - 1) - s->len))
 233                 return 0;
 234
 235         memcpy(s->buffer + s->len, mem, len);
 236         s->len += len;
 237
 238         return len;
 239 }
 240
 241 static notrace void
 242 trace_seq_reset(struct trace_seq *s)
 243 {
 244         s->len = 0;
 245 }
 246
 247 static notrace void
 248 trace_print_seq(struct seq_file *m, struct trace_seq *s)
 249 {
 250         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
 251
 252         s->buffer[len] = 0;
 253         seq_puts(m, s->buffer);
 254
 255         trace_seq_reset(s);
 256 }
 257
 258 notrace static void
 259 flip_trace(struct trace_array_cpu *tr1, struct trace_array_cpu *tr2)
 260 {
 261         struct list_head flip_pages;
 262
 263         INIT_LIST_HEAD(&flip_pages);
 264
 265         memcpy(&tr1->trace_head_idx, &tr2->trace_head_idx,
 266                 sizeof(struct trace_array_cpu) -
 267                 offsetof(struct trace_array_cpu, trace_head_idx));
 268
 269         check_pages(tr1);
 270         check_pages(tr2);
 271         list_splice_init(&tr1->trace_pages, &flip_pages);
 272         list_splice_init(&tr2->trace_pages, &tr1->trace_pages);
 273         list_splice_init(&flip_pages, &tr2->trace_pages);
 274         BUG_ON(!list_empty(&flip_pages));
 275         check_pages(tr1);
 276         check_pages(tr2);
 277 }
 278
 279 notrace void
 280 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
 281 {
 282         struct trace_array_cpu *data;
 283         int i;
 284
 285         WARN_ON_ONCE(!irqs_disabled());
 286         spin_lock(&ftrace_max_lock);
 287         /* clear out all the previous traces */
 288         for_each_possible_cpu(i) {
 289                 data = tr->data[i];
 290                 flip_trace(max_tr.data[i], data);
 291                 tracing_reset(data);
 292         }
 293
 294         __update_max_tr(tr, tsk, cpu);
 295         spin_unlock(&ftrace_max_lock);
 296 }
 297
 298 /**
 299  * update_max_tr_single - only copy one trace over, and reset the rest
 300  * @tr - tracer
 301  * @tsk - task with the latency
 302  * @cpu - the cpu of the buffer to copy.
 303  */
 304 notrace void
 305 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
 306 {
 307         struct trace_array_cpu *data = tr->data[cpu];
 308         int i;
 309
 310         WARN_ON_ONCE(!irqs_disabled());
 311         spin_lock(&ftrace_max_lock);
 312         for_each_possible_cpu(i)
 313                 tracing_reset(max_tr.data[i]);
 314
 315         flip_trace(max_tr.data[cpu], data);
 316         tracing_reset(data);
 317
 318         __update_max_tr(tr, tsk, cpu);
 319         spin_unlock(&ftrace_max_lock);
 320 }
 321
 322 int register_tracer(struct tracer *type)
 323 {
 324         struct tracer *t;
 325         int len;
 326         int ret = 0;
 327
 328         if (!type->name) {
 329                 pr_info("Tracer must have a name\n");
 330                 return -1;
 331         }
 332
 333         mutex_lock(&trace_types_lock);
 334         for (t = trace_types; t; t = t->next) {
 335                 if (strcmp(type->name, t->name) == 0) {
 336                         /* already found */
 337                         pr_info("Trace %s already registered\n",
 338                                 type->name);
 339                         ret = -1;
 340                         goto out;
 341                 }
 342         }
 343
 344 #ifdef CONFIG_FTRACE_STARTUP_TEST
 345         if (type->selftest) {
 346                 struct tracer *saved_tracer = current_trace;
 347                 struct trace_array_cpu *data;
 348                 struct trace_array *tr = &global_trace;
 349                 int saved_ctrl = tr->ctrl;
 350                 int i;
 351                 /*
 352                  * Run a selftest on this tracer.
 353                  * Here we reset the trace buffer, and set the current
 354                  * tracer to be this tracer. The tracer can then run some
 355                  * internal tracing to verify that everything is in order.
 356                  * If we fail, we do not register this tracer.
 357                  */
 358                 for_each_possible_cpu(i) {
 359                         data = tr->data[i];
 360                         if (!head_page(data))
 361                                 continue;
 362                         tracing_reset(data);
 363                 }
 364                 current_trace = type;
 365                 tr->ctrl = 0;
 366                 /* the test is responsible for initializing and enabling */
 367                 pr_info("Testing tracer %s: ", type->name);
 368                 ret = type->selftest(type, tr);
 369                 /* the test is responsible for resetting too */
 370                 current_trace = saved_tracer;
 371                 tr->ctrl = saved_ctrl;
 372                 if (ret) {
 373                         printk(KERN_CONT "FAILED!\n");
 374                         goto out;
 375                 }
 376                 /* Only reset on passing, to avoid touching corrupted buffers */
 377                 for_each_possible_cpu(i) {
 378                         data = tr->data[i];
 379                         if (!head_page(data))
 380                                 continue;
 381                         tracing_reset(data);
 382                 }
 383                 printk(KERN_CONT "PASSED\n");
 384         }
 385 #endif
 386
 387         type->next = trace_types;
 388         trace_types = type;
 389         len = strlen(type->name);
 390         if (len > max_tracer_type_len)
 391                 max_tracer_type_len = len;
 392
 393  out:
 394         mutex_unlock(&trace_types_lock);
 395
 396         return ret;
 397 }
 398
 399 void unregister_tracer(struct tracer *type)
 400 {
 401         struct tracer **t;
 402         int len;
 403
 404         mutex_lock(&trace_types_lock);
 405         for (t = &trace_types; *t; t = &(*t)->next) {
 406                 if (*t == type)
 407                         goto found;
 408         }
 409         pr_info("Trace %s not registered\n", type->name);
 410         goto out;
 411
 412  found:
 413         *t = (*t)->next;
 414         if (strlen(type->name) != max_tracer_type_len)
 415                 goto out;
 416
 417         max_tracer_type_len = 0;
 418         for (t = &trace_types; *t; t = &(*t)->next) {
 419                 len = strlen((*t)->name);
 420                 if (len > max_tracer_type_len)
 421                         max_tracer_type_len = len;
 422         }
 423  out:
 424         mutex_unlock(&trace_types_lock);
 425 }
 426
 427 notrace void tracing_reset(struct trace_array_cpu *data)
 428 {
 429         data->trace_idx = 0;
 430         data->trace_head = data->trace_tail = head_page(data);
 431         data->trace_head_idx = 0;
 432         data->trace_tail_idx = 0;
 433 }
 434
 435 #ifdef CONFIG_FTRACE
 436 static notrace void
 437 function_trace_call(unsigned long ip, unsigned long parent_ip)
 438 {
 439         struct trace_array *tr = &global_trace;
 440         struct trace_array_cpu *data;
 441         unsigned long flags;
 442         long disabled;
 443         int cpu;
 444
 445         if (unlikely(!tracer_enabled))
 446                 return;
 447
 448         local_irq_save(flags);
 449         cpu = raw_smp_processor_id();
 450         data = tr->data[cpu];
 451         disabled = atomic_inc_return(&data->disabled);
 452
 453         if (likely(disabled == 1))
 454                 ftrace(tr, data, ip, parent_ip, flags);
 455
 456         atomic_dec(&data->disabled);
 457         local_irq_restore(flags);
 458 }
 459
 460 static struct ftrace_ops trace_ops __read_mostly =
 461 {
 462         .func = function_trace_call,
 463 };
 464 #endif
 465
 466 notrace void tracing_start_function_trace(void)
 467 {
 468         register_ftrace_function(&trace_ops);
 469 }
 470
 471 notrace void tracing_stop_function_trace(void)
 472 {
 473         unregister_ftrace_function(&trace_ops);
 474 }
 475
 476 #define SAVED_CMDLINES 128
 477 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
 478 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
 479 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
 480 static int cmdline_idx;
 481 static DEFINE_SPINLOCK(trace_cmdline_lock);
 482 atomic_t trace_record_cmdline_disabled;
 483
 484 static void trace_init_cmdlines(void)
 485 {
 486         memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
 487         memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
 488         cmdline_idx = 0;
 489 }
 490
 491 notrace void trace_stop_cmdline_recording(void);
 492
 493 static notrace void trace_save_cmdline(struct task_struct *tsk)
 494 {
 495         unsigned map;
 496         unsigned idx;
 497
 498         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
 499                 return;
 500
 501         /*
 502          * It's not the end of the world if we don't get
 503          * the lock, but we also don't want to spin
 504          * nor do we want to disable interrupts,
 505          * so if we miss here, then better luck next time.
 506          */
 507         if (!spin_trylock(&trace_cmdline_lock))
 508                 return;
 509
 510         idx = map_pid_to_cmdline[tsk->pid];
 511         if (idx >= SAVED_CMDLINES) {
 512                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
 513
 514                 map = map_cmdline_to_pid[idx];
 515                 if (map <= PID_MAX_DEFAULT)
 516                         map_pid_to_cmdline[map] = (unsigned)-1;
 517
 518                 map_pid_to_cmdline[tsk->pid] = idx;
 519
 520                 cmdline_idx = idx;
 521         }
 522
 523         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
 524
 525         spin_unlock(&trace_cmdline_lock);
 526 }
 527
 528 static notrace char *trace_find_cmdline(int pid)
 529 {
 530         char *cmdline = "<...>";
 531         unsigned map;
 532
 533         if (!pid)
 534                 return "<idle>";
 535
 536         if (pid > PID_MAX_DEFAULT)
 537                 goto out;
 538
 539         map = map_pid_to_cmdline[pid];
 540         if (map >= SAVED_CMDLINES)
 541                 goto out;
 542
 543         cmdline = saved_cmdlines[map];
 544
 545  out:
 546         return cmdline;
 547 }
 548
 549 notrace void tracing_record_cmdline(struct task_struct *tsk)
 550 {
 551         if (atomic_read(&trace_record_cmdline_disabled))
 552                 return;
 553
 554         trace_save_cmdline(tsk);
 555 }
 556
 557 static inline notrace struct list_head *
 558 trace_next_list(struct trace_array_cpu *data, struct list_head *next)
 559 {
 560         /*
 561          * Roundrobin - but skip the head (which is not a real page):
 562          */
 563         next = next->next;
 564         if (unlikely(next == &data->trace_pages))
 565                 next = next->next;
 566         BUG_ON(next == &data->trace_pages);
 567
 568         return next;
 569 }
 570
 571 static inline notrace void *
 572 trace_next_page(struct trace_array_cpu *data, void *addr)
 573 {
 574         struct list_head *next;
 575         struct page *page;
 576
 577         page = virt_to_page(addr);
 578
 579         next = trace_next_list(data, &page->lru);
 580         page = list_entry(next, struct page, lru);
 581
 582         return page_address(page);
 583 }
 584
 585 static inline notrace struct trace_entry *
 586 tracing_get_trace_entry(struct trace_array *tr, struct trace_array_cpu *data)
 587 {
 588         unsigned long idx, idx_next;
 589         struct trace_entry *entry;
 590
 591         data->trace_idx++;
 592         idx = data->trace_head_idx;
 593         idx_next = idx + 1;
 594
 595         BUG_ON(idx * TRACE_ENTRY_SIZE >= PAGE_SIZE);
 596
 597         entry = data->trace_head + idx * TRACE_ENTRY_SIZE;
 598
 599         if (unlikely(idx_next >= ENTRIES_PER_PAGE)) {
 600                 data->trace_head = trace_next_page(data, data->trace_head);
 601                 idx_next = 0;
 602         }
 603
 604         if (data->trace_head == data->trace_tail &&
 605             idx_next == data->trace_tail_idx) {
 606                 /* overrun */
 607                 data->trace_tail_idx++;
 608                 if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
 609                         data->trace_tail =
 610                                 trace_next_page(data, data->trace_tail);
 611                         data->trace_tail_idx = 0;
 612                 }
 613         }
 614
 615         data->trace_head_idx = idx_next;
 616
 617         return entry;
 618 }
 619
 620 static inline notrace void
 621 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags)
 622 {
 623         struct task_struct *tsk = current;
 624         unsigned long pc;
 625
 626         pc = preempt_count();
 627
 628         entry->preempt_count    = pc & 0xff;
 629         entry->pid              = tsk->pid;
 630         entry->t                = ftrace_now(raw_smp_processor_id());
 631         entry->flags = (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
 632                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
 633                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
 634                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
 635 }
 636
 637 notrace void
 638 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
 639        unsigned long ip, unsigned long parent_ip, unsigned long flags)
 640 {
 641         struct trace_entry *entry;
 642         unsigned long irq_flags;
 643
 644         spin_lock_irqsave(&data->lock, irq_flags);
 645         entry                   = tracing_get_trace_entry(tr, data);
 646         tracing_generic_entry_update(entry, flags);
 647         entry->type             = TRACE_FN;
 648         entry->fn.ip            = ip;
 649         entry->fn.parent_ip     = parent_ip;
 650         spin_unlock_irqrestore(&data->lock, irq_flags);
 651 }
 652
 653 notrace void
 654 trace_special(struct trace_array *tr, struct trace_array_cpu *data,
 655               unsigned long arg1, unsigned long arg2, unsigned long arg3)
 656 {
 657         struct trace_entry *entry;
 658         unsigned long irq_flags;
 659
 660         spin_lock_irqsave(&data->lock, irq_flags);
 661         entry                   = tracing_get_trace_entry(tr, data);
 662         tracing_generic_entry_update(entry, 0);
 663         entry->type             = TRACE_SPECIAL;
 664         entry->special.arg1     = arg1;
 665         entry->special.arg2     = arg2;
 666         entry->special.arg3     = arg3;
 667         spin_unlock_irqrestore(&data->lock, irq_flags);
 668 }
 669
 670 notrace void
 671 tracing_sched_switch_trace(struct trace_array *tr,
 672                            struct trace_array_cpu *data,
 673                            struct task_struct *prev, struct task_struct *next,
 674                            unsigned long flags)
 675 {
 676         struct trace_entry *entry;
 677         unsigned long irq_flags;
 678
 679         spin_lock_irqsave(&data->lock, irq_flags);
 680         entry                   = tracing_get_trace_entry(tr, data);
 681         tracing_generic_entry_update(entry, flags);
 682         entry->type             = TRACE_CTX;
 683         entry->ctx.prev_pid     = prev->pid;
 684         entry->ctx.prev_prio    = prev->prio;
 685         entry->ctx.prev_state   = prev->state;
 686         entry->ctx.next_pid     = next->pid;
 687         entry->ctx.next_prio    = next->prio;
 688         spin_unlock_irqrestore(&data->lock, irq_flags);
 689 }
 690
 691 enum trace_file_type {
 692         TRACE_FILE_LAT_FMT      = 1,
 693 };
 694
 695 static struct trace_entry *
 696 trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
 697                 struct trace_iterator *iter, int cpu)
 698 {
 699         struct page *page;
 700         struct trace_entry *array;
 701
 702         if (iter->next_idx[cpu] >= tr->entries ||
 703             iter->next_idx[cpu] >= data->trace_idx ||
 704             (data->trace_head == data->trace_tail &&
 705              data->trace_head_idx == data->trace_tail_idx))
 706                 return NULL;
 707
 708         if (!iter->next_page[cpu]) {
 709                 /* Initialize the iterator for this cpu trace buffer */
 710                 WARN_ON(!data->trace_tail);
 711                 page = virt_to_page(data->trace_tail);
 712                 iter->next_page[cpu] = &page->lru;
 713                 iter->next_page_idx[cpu] = data->trace_tail_idx;
 714         }
 715
 716         page = list_entry(iter->next_page[cpu], struct page, lru);
 717         BUG_ON(&data->trace_pages == &page->lru);
 718
 719         array = page_address(page);
 720
 721         WARN_ON(iter->next_page_idx[cpu] >= ENTRIES_PER_PAGE);
 722         return &array[iter->next_page_idx[cpu]];
 723 }
 724
 725 static struct notrace trace_entry *
 726 find_next_entry(struct trace_iterator *iter, int *ent_cpu)
 727 {
 728         struct trace_array *tr = iter->tr;
 729         struct trace_entry *ent, *next = NULL;
 730         int next_cpu = -1;
 731         int cpu;
 732
 733         for_each_possible_cpu(cpu) {
 734                 if (!head_page(tr->data[cpu]))
 735                         continue;
 736                 ent = trace_entry_idx(tr, tr->data[cpu], iter, cpu);
 737                 /*
 738                  * Pick the entry with the smallest timestamp:
 739                  */
 740                 if (ent && (!next || ent->t < next->t)) {
 741                         next = ent;
 742                         next_cpu = cpu;
 743                 }
 744         }
 745
 746         if (ent_cpu)
 747                 *ent_cpu = next_cpu;
 748
 749         return next;
 750 }
 751
 752 static notrace void trace_iterator_increment(struct trace_iterator *iter)
 753 {
 754         iter->idx++;
 755         iter->next_idx[iter->cpu]++;
 756         iter->next_page_idx[iter->cpu]++;
 757
 758         if (iter->next_page_idx[iter->cpu] >= ENTRIES_PER_PAGE) {
 759                 struct trace_array_cpu *data = iter->tr->data[iter->cpu];
 760
 761                 iter->next_page_idx[iter->cpu] = 0;
 762                 iter->next_page[iter->cpu] =
 763                         trace_next_list(data, iter->next_page[iter->cpu]);
 764         }
 765 }
 766
 767 static notrace void trace_consume(struct trace_iterator *iter)
 768 {
 769         struct trace_array_cpu *data = iter->tr->data[iter->cpu];
 770
 771         data->trace_tail_idx++;
 772         if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
 773                 data->trace_tail = trace_next_page(data, data->trace_tail);
 774                 data->trace_tail_idx = 0;
 775         }
 776
 777         /* Check if we empty it, then reset the index */
 778         if (data->trace_head == data->trace_tail &&
 779             data->trace_head_idx == data->trace_tail_idx)
 780                 data->trace_idx = 0;
 781 }
 782
 783 static notrace void *find_next_entry_inc(struct trace_iterator *iter)
 784 {
 785         struct trace_entry *next;
 786         int next_cpu = -1;
 787
 788         next = find_next_entry(iter, &next_cpu);
 789
 790         iter->prev_ent = iter->ent;
 791         iter->prev_cpu = iter->cpu;
 792
 793         iter->ent = next;
 794         iter->cpu = next_cpu;
 795
 796         if (next)
 797                 trace_iterator_increment(iter);
 798
 799         return next ? iter : NULL;
 800 }
 801
 802 static notrace void *s_next(struct seq_file *m, void *v, loff_t *pos)
 803 {
 804         struct trace_iterator *iter = m->private;
 805         void *last_ent = iter->ent;
 806         int i = (int)*pos;
 807         void *ent;
 808
 809         (*pos)++;
 810
 811         /* can't go backwards */
 812         if (iter->idx > i)
 813                 return NULL;
 814
 815         if (iter->idx < 0)
 816                 ent = find_next_entry_inc(iter);
 817         else
 818                 ent = iter;
 819
 820         while (ent && iter->idx < i)
 821                 ent = find_next_entry_inc(iter);
 822
 823         iter->pos = *pos;
 824
 825         if (last_ent && !ent)
 826                 seq_puts(m, "\n\nvim:ft=help\n");
 827
 828         return ent;
 829 }
 830
 831 static void *s_start(struct seq_file *m, loff_t *pos)
 832 {
 833         struct trace_iterator *iter = m->private;
 834         void *p = NULL;
 835         loff_t l = 0;
 836         int i;
 837
 838         mutex_lock(&trace_types_lock);
 839
 840         if (!current_trace || current_trace != iter->trace)
 841                 return NULL;
 842
 843         atomic_inc(&trace_record_cmdline_disabled);
 844
 845         /* let the tracer grab locks here if needed */
 846         if (current_trace->start)
 847                 current_trace->start(iter);
 848
 849         if (*pos != iter->pos) {
 850                 iter->ent = NULL;
 851                 iter->cpu = 0;
 852                 iter->idx = -1;
 853                 iter->prev_ent = NULL;
 854                 iter->prev_cpu = -1;
 855
 856                 for_each_possible_cpu(i) {
 857                         iter->next_idx[i] = 0;
 858                         iter->next_page[i] = NULL;
 859                 }
 860
 861                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
 862                         ;
 863
 864         } else {
 865                 l = *pos - 1;
 866                 p = s_next(m, p, &l);
 867         }
 868
 869         return p;
 870 }
 871
 872 static void s_stop(struct seq_file *m, void *p)
 873 {
 874         struct trace_iterator *iter = m->private;
 875
 876         atomic_dec(&trace_record_cmdline_disabled);
 877
 878         /* let the tracer release locks here if needed */
 879         if (current_trace && current_trace == iter->trace && iter->trace->stop)
 880                 iter->trace->stop(iter);
 881
 882         mutex_unlock(&trace_types_lock);
 883 }
 884
 885 static int
 886 seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
 887 {
 888 #ifdef CONFIG_KALLSYMS
 889         char str[KSYM_SYMBOL_LEN];
 890
 891         kallsyms_lookup(address, NULL, NULL, NULL, str);
 892
 893         return trace_seq_printf(s, fmt, str);
 894 #endif
 895         return 1;
 896 }
 897
 898 static int
 899 seq_print_sym_offset(struct trace_seq *s, const char *fmt,
 900                      unsigned long address)
 901 {
 902 #ifdef CONFIG_KALLSYMS
 903         char str[KSYM_SYMBOL_LEN];
 904
 905         sprint_symbol(str, address);
 906         return trace_seq_printf(s, fmt, str);
 907 #endif
 908         return 1;
 909 }
 910
 911 #ifndef CONFIG_64BIT
 912 # define IP_FMT "%08lx"
 913 #else
 914 # define IP_FMT "%016lx"
 915 #endif
 916
 917 static notrace int
 918 seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
 919 {
 920         int ret;
 921
 922         if (!ip)
 923                 return trace_seq_printf(s, "0");
 924
 925         if (sym_flags & TRACE_ITER_SYM_OFFSET)
 926                 ret = seq_print_sym_offset(s, "%s", ip);
 927         else
 928                 ret = seq_print_sym_short(s, "%s", ip);
 929
 930         if (!ret)
 931                 return 0;
 932
 933         if (sym_flags & TRACE_ITER_SYM_ADDR)
 934                 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
 935         return ret;
 936 }
 937
 938 static notrace void print_lat_help_header(struct seq_file *m)
 939 {
 940         seq_puts(m, "#                _------=> CPU#            \n");
 941         seq_puts(m, "#               / _-----=> irqs-off        \n");
 942         seq_puts(m, "#              | / _----=> need-resched    \n");
 943         seq_puts(m, "#              || / _---=> hardirq/softirq \n");
 944         seq_puts(m, "#              ||| / _--=> preempt-depth   \n");
 945         seq_puts(m, "#              |||| /                      \n");
 946         seq_puts(m, "#              |||||     delay             \n");
 947         seq_puts(m, "#  cmd     pid ||||| time  |   caller      \n");
 948         seq_puts(m, "#     \\   /    |||||   \\   |   /           \n");
 949 }
 950
 951 static notrace void print_func_help_header(struct seq_file *m)
 952 {
 953         seq_puts(m, "#           TASK-PID   CPU#    TIMESTAMP  FUNCTION\n");
 954         seq_puts(m, "#              | |      |          |         |\n");
 955 }
 956
 957
 958 static notrace void
 959 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
 960 {
 961         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
 962         struct trace_array *tr = iter->tr;
 963         struct trace_array_cpu *data = tr->data[tr->cpu];
 964         struct tracer *type = current_trace;
 965         unsigned long total   = 0;
 966         unsigned long entries = 0;
 967         int cpu;
 968         const char *name = "preemption";
 969
 970         if (type)
 971                 name = type->name;
 972
 973         for_each_possible_cpu(cpu) {
 974                 if (head_page(tr->data[cpu])) {
 975                         total += tr->data[cpu]->trace_idx;
 976                         if (tr->data[cpu]->trace_idx > tr->entries)
 977                                 entries += tr->entries;
 978                         else
 979                                 entries += tr->data[cpu]->trace_idx;
 980                 }
 981         }
 982
 983         seq_printf(m, "%s latency trace v1.1.5 on %s\n",
 984                    name, UTS_RELEASE);
 985         seq_puts(m, "-----------------------------------"
 986                  "---------------------------------\n");
 987         seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
 988                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
 989                    nsecs_to_usecs(data->saved_latency),
 990                    entries,
 991                    total,
 992                    tr->cpu,
 993 #if defined(CONFIG_PREEMPT_NONE)
 994                    "server",
 995 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
 996                    "desktop",
 997 #elif defined(CONFIG_PREEMPT_DESKTOP)
 998                    "preempt",
 999 #else
1000                    "unknown",
1001 #endif
1002                    /* These are reserved for later use */
1003                    0, 0, 0, 0);
1004 #ifdef CONFIG_SMP
1005         seq_printf(m, " #P:%d)\n", num_online_cpus());
1006 #else
1007         seq_puts(m, ")\n");
1008 #endif
1009         seq_puts(m, "    -----------------\n");
1010         seq_printf(m, "    | task: %.16s-%d "
1011                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1012                    data->comm, data->pid, data->uid, data->nice,
1013                    data->policy, data->rt_priority);
1014         seq_puts(m, "    -----------------\n");
1015
1016         if (data->critical_start) {
1017                 seq_puts(m, " => started at: ");
1018                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1019                 trace_print_seq(m, &iter->seq);
1020                 seq_puts(m, "\n => ended at:   ");
1021                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1022                 trace_print_seq(m, &iter->seq);
1023                 seq_puts(m, "\n");
1024         }
1025
1026         seq_puts(m, "\n");
1027 }
1028
1029 static notrace void
1030 lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1031 {
1032         int hardirq, softirq;
1033         char *comm;
1034
1035         comm = trace_find_cmdline(entry->pid);
1036
1037         trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1038         trace_seq_printf(s, "%d", cpu);
1039         trace_seq_printf(s, "%c%c",
1040                         (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : '.',
1041                         ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1042
1043         hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1044         softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1045         if (hardirq && softirq)
1046                 trace_seq_putc(s, 'H');
1047         else {
1048                 if (hardirq)
1049                         trace_seq_putc(s, 'h');
1050                 else {
1051                         if (softirq)
1052                                 trace_seq_putc(s, 's');
1053                         else
1054                                 trace_seq_putc(s, '.');
1055                 }
1056         }
1057
1058         if (entry->preempt_count)
1059                 trace_seq_printf(s, "%x", entry->preempt_count);
1060         else
1061                 trace_seq_puts(s, ".");
1062 }
1063
1064 unsigned long preempt_mark_thresh = 100;
1065
1066 static notrace void
1067 lat_print_timestamp(struct trace_seq *s, unsigned long long abs_usecs,
1068                     unsigned long rel_usecs)
1069 {
1070         trace_seq_printf(s, " %4lldus", abs_usecs);
1071         if (rel_usecs > preempt_mark_thresh)
1072                 trace_seq_puts(s, "!: ");
1073         else if (rel_usecs > 1)
1074                 trace_seq_puts(s, "+: ");
1075         else
1076                 trace_seq_puts(s, " : ");
1077 }
1078
1079 static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1080
1081 static notrace int
1082 print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1083 {
1084         struct trace_seq *s = &iter->seq;
1085         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1086         struct trace_entry *next_entry = find_next_entry(iter, NULL);
1087         unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1088         struct trace_entry *entry = iter->ent;
1089         unsigned long abs_usecs;
1090         unsigned long rel_usecs;
1091         char *comm;
1092         int S;
1093
1094         if (!next_entry)
1095                 next_entry = entry;
1096         rel_usecs = ns2usecs(next_entry->t - entry->t);
1097         abs_usecs = ns2usecs(entry->t - iter->tr->time_start);
1098
1099         if (verbose) {
1100                 comm = trace_find_cmdline(entry->pid);
1101                 trace_seq_printf(s, "%16s %5d %d %d %08x %08x [%08lx]"
1102                                  " %ld.%03ldms (+%ld.%03ldms): ",
1103                                  comm,
1104                                  entry->pid, cpu, entry->flags,
1105                                  entry->preempt_count, trace_idx,
1106                                  ns2usecs(entry->t),
1107                                  abs_usecs/1000,
1108                                  abs_usecs % 1000, rel_usecs/1000,
1109                                  rel_usecs % 1000);
1110         } else {
1111                 lat_print_generic(s, entry, cpu);
1112                 lat_print_timestamp(s, abs_usecs, rel_usecs);
1113         }
1114         switch (entry->type) {
1115         case TRACE_FN:
1116                 seq_print_ip_sym(s, entry->fn.ip, sym_flags);
1117                 trace_seq_puts(s, " (");
1118                 seq_print_ip_sym(s, entry->fn.parent_ip, sym_flags);
1119                 trace_seq_puts(s, ")\n");
1120                 break;
1121         case TRACE_CTX:
1122                 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1123                         state_to_char[entry->ctx.prev_state] : 'X';
1124                 comm = trace_find_cmdline(entry->ctx.next_pid);
1125                 trace_seq_printf(s, " %d:%d:%c --> %d:%d %s\n",
1126                                  entry->ctx.prev_pid,
1127                                  entry->ctx.prev_prio,
1128                                  S,
1129                                  entry->ctx.next_pid,
1130                                  entry->ctx.next_prio,
1131                                  comm);
1132                 break;
1133         case TRACE_SPECIAL:
1134                 trace_seq_printf(s, " %lx %lx %lx\n",
1135                                  entry->special.arg1,
1136                                  entry->special.arg2,
1137                                  entry->special.arg3);
1138                 break;
1139         default:
1140                 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1141         }
1142         return 1;
1143 }
1144
1145 static notrace int print_trace_fmt(struct trace_iterator *iter)
1146 {
1147         struct trace_seq *s = &iter->seq;
1148         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1149         struct trace_entry *entry;
1150         unsigned long usec_rem;
1151         unsigned long long t;
1152         unsigned long secs;
1153         char *comm;
1154         int S;
1155         int ret;
1156
1157         entry = iter->ent;
1158
1159         comm = trace_find_cmdline(iter->ent->pid);
1160
1161         t = ns2usecs(entry->t);
1162         usec_rem = do_div(t, 1000000ULL);
1163         secs = (unsigned long)t;
1164
1165         ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1166         if (!ret)
1167                 return 0;
1168         ret = trace_seq_printf(s, "[%02d] ", iter->cpu);
1169         if (!ret)
1170                 return 0;
1171         ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1172         if (!ret)
1173                 return 0;
1174
1175         switch (entry->type) {
1176         case TRACE_FN:
1177                 ret = seq_print_ip_sym(s, entry->fn.ip, sym_flags);
1178                 if (!ret)
1179                         return 0;
1180                 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1181                                                 entry->fn.parent_ip) {
1182                         ret = trace_seq_printf(s, " <-");
1183                         if (!ret)
1184                                 return 0;
1185                         ret = seq_print_ip_sym(s, entry->fn.parent_ip,
1186                                                sym_flags);
1187                         if (!ret)
1188                                 return 0;
1189                 }
1190                 ret = trace_seq_printf(s, "\n");
1191                 if (!ret)
1192                         return 0;
1193                 break;
1194         case TRACE_CTX:
1195                 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1196                         state_to_char[entry->ctx.prev_state] : 'X';
1197                 ret = trace_seq_printf(s, " %d:%d:%c ==> %d:%d\n",
1198                                        entry->ctx.prev_pid,
1199                                        entry->ctx.prev_prio,
1200                                        S,
1201                                        entry->ctx.next_pid,
1202                                        entry->ctx.next_prio);
1203                 if (!ret)
1204                         return 0;
1205                 break;
1206         case TRACE_SPECIAL:
1207                 ret = trace_seq_printf(s, " %lx %lx %lx\n",
1208                                  entry->special.arg1,
1209                                  entry->special.arg2,
1210                                  entry->special.arg3);
1211                 if (!ret)
1212                         return 0;
1213                 break;
1214         }
1215         return 1;
1216 }
1217
1218 static notrace int print_raw_fmt(struct trace_iterator *iter)
1219 {
1220         struct trace_seq *s = &iter->seq;
1221         struct trace_entry *entry;
1222         int ret;
1223         int S;
1224
1225         entry = iter->ent;
1226
1227         ret = trace_seq_printf(s, "%d %d %llu ",
1228                 entry->pid, iter->cpu, entry->t);
1229         if (!ret)
1230                 return 0;
1231
1232         switch (entry->type) {
1233         case TRACE_FN:
1234                 ret = trace_seq_printf(s, "%x %x\n",
1235                                         entry->fn.ip, entry->fn.parent_ip);
1236                 if (!ret)
1237                         return 0;
1238                 break;
1239         case TRACE_CTX:
1240                 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1241                         state_to_char[entry->ctx.prev_state] : 'X';
1242                 ret = trace_seq_printf(s, "%d %d %c %d %d\n",
1243                                        entry->ctx.prev_pid,
1244                                        entry->ctx.prev_prio,
1245                                        S,
1246                                        entry->ctx.next_pid,
1247                                        entry->ctx.next_prio);
1248                 if (!ret)
1249                         return 0;
1250                 break;
1251         case TRACE_SPECIAL:
1252                 ret = trace_seq_printf(s, " %lx %lx %lx\n",
1253                                  entry->special.arg1,
1254                                  entry->special.arg2,
1255                                  entry->special.arg3);
1256                 if (!ret)
1257                         return 0;
1258                 break;
1259         }
1260         return 1;
1261 }
1262
1263 #define SEQ_PUT_FIELD_RET(s, x)                         \
1264 do {                                                    \
1265         if (!trace_seq_putmem(s, &(x), sizeof(x)))      \
1266                 return 0;                               \
1267 } while (0)
1268
1269 static notrace int print_bin_fmt(struct trace_iterator *iter)
1270 {
1271         struct trace_seq *s = &iter->seq;
1272         struct trace_entry *entry;
1273
1274         entry = iter->ent;
1275
1276         SEQ_PUT_FIELD_RET(s, entry->pid);
1277         SEQ_PUT_FIELD_RET(s, entry->cpu);
1278         SEQ_PUT_FIELD_RET(s, entry->t);
1279
1280         switch (entry->type) {
1281         case TRACE_FN:
1282                 SEQ_PUT_FIELD_RET(s, entry->fn.ip);
1283                 SEQ_PUT_FIELD_RET(s, entry->fn.parent_ip);
1284                 break;
1285         case TRACE_CTX:
1286                 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_pid);
1287                 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_prio);
1288                 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_state);
1289                 SEQ_PUT_FIELD_RET(s, entry->ctx.next_pid);
1290                 SEQ_PUT_FIELD_RET(s, entry->ctx.next_prio);
1291                 break;
1292         case TRACE_SPECIAL:
1293                 SEQ_PUT_FIELD_RET(s, entry->special.arg1);
1294                 SEQ_PUT_FIELD_RET(s, entry->special.arg2);
1295                 SEQ_PUT_FIELD_RET(s, entry->special.arg3);
1296                 break;
1297         }
1298         return 1;
1299 }
1300
1301 static int trace_empty(struct trace_iterator *iter)
1302 {
1303         struct trace_array_cpu *data;
1304         int cpu;
1305
1306         for_each_possible_cpu(cpu) {
1307                 data = iter->tr->data[cpu];
1308
1309                 if (head_page(data) && data->trace_idx &&
1310                     (data->trace_tail != data->trace_head ||
1311                      data->trace_tail_idx != data->trace_head_idx))
1312                         return 0;
1313         }
1314         return 1;
1315 }
1316
1317 static int print_trace_line(struct trace_iterator *iter)
1318 {
1319         if (trace_flags & TRACE_ITER_BIN)
1320                 return print_bin_fmt(iter);
1321
1322         if (trace_flags & TRACE_ITER_RAW)
1323                 return print_raw_fmt(iter);
1324
1325         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1326                 return print_lat_fmt(iter, iter->idx, iter->cpu);
1327
1328         return print_trace_fmt(iter);
1329 }
1330
1331 static int s_show(struct seq_file *m, void *v)
1332 {
1333         struct trace_iterator *iter = v;
1334
1335         if (iter->ent == NULL) {
1336                 if (iter->tr) {
1337                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1338                         seq_puts(m, "#\n");
1339                 }
1340                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1341                         /* print nothing if the buffers are empty */
1342                         if (trace_empty(iter))
1343                                 return 0;
1344                         print_trace_header(m, iter);
1345                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1346                                 print_lat_help_header(m);
1347                 } else {
1348                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1349                                 print_func_help_header(m);
1350                 }
1351         } else {
1352                 print_trace_line(iter);
1353                 trace_print_seq(m, &iter->seq);
1354         }
1355
1356         return 0;
1357 }
1358
1359 static struct seq_operations tracer_seq_ops = {
1360         .start          = s_start,
1361         .next           = s_next,
1362         .stop           = s_stop,
1363         .show           = s_show,
1364 };
1365
1366 static struct trace_iterator notrace *
1367 __tracing_open(struct inode *inode, struct file *file, int *ret)
1368 {
1369         struct trace_iterator *iter;
1370
1371         if (tracing_disabled) {
1372                 *ret = -ENODEV;
1373                 return NULL;
1374         }
1375
1376         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1377         if (!iter) {
1378                 *ret = -ENOMEM;
1379                 goto out;
1380         }
1381
1382         mutex_lock(&trace_types_lock);
1383         if (current_trace && current_trace->print_max)
1384                 iter->tr = &max_tr;
1385         else
1386                 iter->tr = inode->i_private;
1387         iter->trace = current_trace;
1388         iter->pos = -1;
1389
1390         /* TODO stop tracer */
1391         *ret = seq_open(file, &tracer_seq_ops);
1392         if (!*ret) {
1393                 struct seq_file *m = file->private_data;
1394                 m->private = iter;
1395
1396                 /* stop the trace while dumping */
1397                 if (iter->tr->ctrl)
1398                         tracer_enabled = 0;
1399
1400                 if (iter->trace && iter->trace->open)
1401                         iter->trace->open(iter);
1402         } else {
1403                 kfree(iter);
1404                 iter = NULL;
1405         }
1406         mutex_unlock(&trace_types_lock);
1407
1408  out:
1409         return iter;
1410 }
1411
1412 int tracing_open_generic(struct inode *inode, struct file *filp)
1413 {
1414         if (tracing_disabled)
1415                 return -ENODEV;
1416
1417         filp->private_data = inode->i_private;
1418         return 0;
1419 }
1420
1421 int tracing_release(struct inode *inode, struct file *file)
1422 {
1423         struct seq_file *m = (struct seq_file *)file->private_data;
1424         struct trace_iterator *iter = m->private;
1425
1426         mutex_lock(&trace_types_lock);
1427         if (iter->trace && iter->trace->close)
1428                 iter->trace->close(iter);
1429
1430         /* reenable tracing if it was previously enabled */
1431         if (iter->tr->ctrl)
1432                 tracer_enabled = 1;
1433         mutex_unlock(&trace_types_lock);
1434
1435         seq_release(inode, file);
1436         kfree(iter);
1437         return 0;
1438 }
1439
1440 static int tracing_open(struct inode *inode, struct file *file)
1441 {
1442         int ret;
1443
1444         __tracing_open(inode, file, &ret);
1445
1446         return ret;
1447 }
1448
1449 static int tracing_lt_open(struct inode *inode, struct file *file)
1450 {
1451         struct trace_iterator *iter;
1452         int ret;
1453
1454         iter = __tracing_open(inode, file, &ret);
1455
1456         if (!ret)
1457                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
1458
1459         return ret;
1460 }
1461
1462
1463 static notrace void *
1464 t_next(struct seq_file *m, void *v, loff_t *pos)
1465 {
1466         struct tracer *t = m->private;
1467
1468         (*pos)++;
1469
1470         if (t)
1471                 t = t->next;
1472
1473         m->private = t;
1474
1475         return t;
1476 }
1477
1478 static void *t_start(struct seq_file *m, loff_t *pos)
1479 {
1480         struct tracer *t = m->private;
1481         loff_t l = 0;
1482
1483         mutex_lock(&trace_types_lock);
1484         for (; t && l < *pos; t = t_next(m, t, &l))
1485                 ;
1486
1487         return t;
1488 }
1489
1490 static void t_stop(struct seq_file *m, void *p)
1491 {
1492         mutex_unlock(&trace_types_lock);
1493 }
1494
1495 static int t_show(struct seq_file *m, void *v)
1496 {
1497         struct tracer *t = v;
1498
1499         if (!t)
1500                 return 0;
1501
1502         seq_printf(m, "%s", t->name);
1503         if (t->next)
1504                 seq_putc(m, ' ');
1505         else
1506                 seq_putc(m, '\n');
1507
1508         return 0;
1509 }
1510
1511 static struct seq_operations show_traces_seq_ops = {
1512         .start          = t_start,
1513         .next           = t_next,
1514         .stop           = t_stop,
1515         .show           = t_show,
1516 };
1517
1518 static int show_traces_open(struct inode *inode, struct file *file)
1519 {
1520         int ret;
1521
1522         if (tracing_disabled)
1523                 return -ENODEV;
1524
1525         ret = seq_open(file, &show_traces_seq_ops);
1526         if (!ret) {
1527                 struct seq_file *m = file->private_data;
1528                 m->private = trace_types;
1529         }
1530
1531         return ret;
1532 }
1533
1534 static struct file_operations tracing_fops = {
1535         .open           = tracing_open,
1536         .read           = seq_read,
1537         .llseek         = seq_lseek,
1538         .release        = tracing_release,
1539 };
1540
1541 static struct file_operations tracing_lt_fops = {
1542         .open           = tracing_lt_open,
1543         .read           = seq_read,
1544         .llseek         = seq_lseek,
1545         .release        = tracing_release,
1546 };
1547
1548 static struct file_operations show_traces_fops = {
1549         .open = show_traces_open,
1550         .read = seq_read,
1551         .release = seq_release,
1552 };
1553
1554 static ssize_t
1555 tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
1556                        size_t cnt, loff_t *ppos)
1557 {
1558         char *buf;
1559         int r = 0;
1560         int len = 0;
1561         int i;
1562
1563         /* calulate max size */
1564         for (i = 0; trace_options[i]; i++) {
1565                 len += strlen(trace_options[i]);
1566                 len += 3; /* "no" and space */
1567         }
1568
1569         /* +2 for \n and \0 */
1570         buf = kmalloc(len + 2, GFP_KERNEL);
1571         if (!buf)
1572                 return -ENOMEM;
1573
1574         for (i = 0; trace_options[i]; i++) {
1575                 if (trace_flags & (1 << i))
1576                         r += sprintf(buf + r, "%s ", trace_options[i]);
1577                 else
1578                         r += sprintf(buf + r, "no%s ", trace_options[i]);
1579         }
1580
1581         r += sprintf(buf + r, "\n");
1582         WARN_ON(r >= len + 2);
1583
1584         r = simple_read_from_buffer(ubuf, cnt, ppos,
1585                                     buf, r);
1586
1587         kfree(buf);
1588
1589         return r;
1590 }
1591
1592 static ssize_t
1593 tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
1594                         size_t cnt, loff_t *ppos)
1595 {
1596         char buf[64];
1597         char *cmp = buf;
1598         int neg = 0;
1599         int i;
1600
1601         if (cnt > 63)
1602                 cnt = 63;
1603
1604         if (copy_from_user(&buf, ubuf, cnt))
1605                 return -EFAULT;
1606
1607         buf[cnt] = 0;
1608
1609         if (strncmp(buf, "no", 2) == 0) {
1610                 neg = 1;
1611                 cmp += 2;
1612         }
1613
1614         for (i = 0; trace_options[i]; i++) {
1615                 int len = strlen(trace_options[i]);
1616
1617                 if (strncmp(cmp, trace_options[i], len) == 0) {
1618                         if (neg)
1619                                 trace_flags &= ~(1 << i);
1620                         else
1621                                 trace_flags |= (1 << i);
1622                         break;
1623                 }
1624         }
1625
1626         filp->f_pos += cnt;
1627
1628         return cnt;
1629 }
1630
1631 static struct file_operations tracing_iter_fops = {
1632         .open = tracing_open_generic,
1633         .read = tracing_iter_ctrl_read,
1634         .write = tracing_iter_ctrl_write,
1635 };
1636
1637 static const char readme_msg[] =
1638         "tracing mini-HOWTO:\n\n"
1639         "# mkdir /debug\n"
1640         "# mount -t debugfs nodev /debug\n\n"
1641         "# cat /debug/tracing/available_tracers\n"
1642         "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
1643         "# cat /debug/tracing/current_tracer\n"
1644         "none\n"
1645         "# echo sched_switch > /debug/tracing/current_tracer\n"
1646         "# cat /debug/tracing/current_tracer\n"
1647         "sched_switch\n"
1648         "# cat /debug/tracing/iter_ctrl\n"
1649         "noprint-parent nosym-offset nosym-addr noverbose\n"
1650         "# echo print-parent > /debug/tracing/iter_ctrl\n"
1651         "# echo 1 > /debug/tracing/tracing_enabled\n"
1652         "# cat /debug/tracing/trace > /tmp/trace.txt\n"
1653         "echo 0 > /debug/tracing/tracing_enabled\n"
1654 ;
1655
1656 static ssize_t
1657 tracing_readme_read(struct file *filp, char __user *ubuf,
1658                        size_t cnt, loff_t *ppos)
1659 {
1660         return simple_read_from_buffer(ubuf, cnt, ppos,
1661                                         readme_msg, strlen(readme_msg));
1662 }
1663
1664 static struct file_operations tracing_readme_fops = {
1665         .open = tracing_open_generic,
1666         .read = tracing_readme_read,
1667 };
1668
1669
1670 static ssize_t
1671 tracing_ctrl_read(struct file *filp, char __user *ubuf,
1672                   size_t cnt, loff_t *ppos)
1673 {
1674         struct trace_array *tr = filp->private_data;
1675         char buf[64];
1676         int r;
1677
1678         r = sprintf(buf, "%ld\n", tr->ctrl);
1679         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1680 }
1681
1682 static ssize_t
1683 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
1684                    size_t cnt, loff_t *ppos)
1685 {
1686         struct trace_array *tr = filp->private_data;
1687         long val;
1688         char buf[64];
1689
1690         if (cnt > 63)
1691                 cnt = 63;
1692
1693         if (copy_from_user(&buf, ubuf, cnt))
1694                 return -EFAULT;
1695
1696         buf[cnt] = 0;
1697
1698         val = simple_strtoul(buf, NULL, 10);
1699
1700         val = !!val;
1701
1702         mutex_lock(&trace_types_lock);
1703         if (tr->ctrl ^ val) {
1704                 if (val)
1705                         tracer_enabled = 1;
1706                 else
1707                         tracer_enabled = 0;
1708
1709                 tr->ctrl = val;
1710
1711                 if (current_trace && current_trace->ctrl_update)
1712                         current_trace->ctrl_update(tr);
1713         }
1714         mutex_unlock(&trace_types_lock);
1715
1716         filp->f_pos += cnt;
1717
1718         return cnt;
1719 }
1720
1721 static ssize_t
1722 tracing_set_trace_read(struct file *filp, char __user *ubuf,
1723                        size_t cnt, loff_t *ppos)
1724 {
1725         char buf[max_tracer_type_len+2];
1726         int r;
1727
1728         mutex_lock(&trace_types_lock);
1729         if (current_trace)
1730                 r = sprintf(buf, "%s\n", current_trace->name);
1731         else
1732                 r = sprintf(buf, "\n");
1733         mutex_unlock(&trace_types_lock);
1734
1735         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1736 }
1737
1738 static ssize_t
1739 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
1740                         size_t cnt, loff_t *ppos)
1741 {
1742         struct trace_array *tr = &global_trace;
1743         struct tracer *t;
1744         char buf[max_tracer_type_len+1];
1745         int i;
1746
1747         if (cnt > max_tracer_type_len)
1748                 cnt = max_tracer_type_len;
1749
1750         if (copy_from_user(&buf, ubuf, cnt))
1751                 return -EFAULT;
1752
1753         buf[cnt] = 0;
1754
1755         /* strip ending whitespace. */
1756         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
1757                 buf[i] = 0;
1758
1759         mutex_lock(&trace_types_lock);
1760         for (t = trace_types; t; t = t->next) {
1761                 if (strcmp(t->name, buf) == 0)
1762                         break;
1763         }
1764         if (!t || t == current_trace)
1765                 goto out;
1766
1767         if (current_trace && current_trace->reset)
1768                 current_trace->reset(tr);
1769
1770         current_trace = t;
1771         if (t->init)
1772                 t->init(tr);
1773
1774  out:
1775         mutex_unlock(&trace_types_lock);
1776
1777         filp->f_pos += cnt;
1778
1779         return cnt;
1780 }
1781
1782 static ssize_t
1783 tracing_max_lat_read(struct file *filp, char __user *ubuf,
1784                      size_t cnt, loff_t *ppos)
1785 {
1786         unsigned long *ptr = filp->private_data;
1787         char buf[64];
1788         int r;
1789
1790         r = snprintf(buf, 64, "%ld\n",
1791                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
1792         if (r > 64)
1793                 r = 64;
1794         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1795 }
1796
1797 static ssize_t
1798 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
1799                       size_t cnt, loff_t *ppos)
1800 {
1801         long *ptr = filp->private_data;
1802         long val;
1803         char buf[64];
1804
1805         if (cnt > 63)
1806                 cnt = 63;
1807
1808         if (copy_from_user(&buf, ubuf, cnt))
1809                 return -EFAULT;
1810
1811         buf[cnt] = 0;
1812
1813         val = simple_strtoul(buf, NULL, 10);
1814
1815         *ptr = val * 1000;
1816
1817         return cnt;
1818 }
1819
1820 static atomic_t tracing_reader;
1821
1822 static int tracing_open_pipe(struct inode *inode, struct file *filp)
1823 {
1824         struct trace_iterator *iter;
1825
1826         if (tracing_disabled)
1827                 return -ENODEV;
1828
1829         /* We only allow for reader of the pipe */
1830         if (atomic_inc_return(&tracing_reader) != 1) {
1831                 atomic_dec(&tracing_reader);
1832                 return -EBUSY;
1833         }
1834
1835         /* create a buffer to store the information to pass to userspace */
1836         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1837         if (!iter)
1838                 return -ENOMEM;
1839
1840         iter->tr = &global_trace;
1841
1842         filp->private_data = iter;
1843
1844         return 0;
1845 }
1846
1847 static int tracing_release_pipe(struct inode *inode, struct file *file)
1848 {
1849         struct trace_iterator *iter = file->private_data;
1850
1851         kfree(iter);
1852         atomic_dec(&tracing_reader);
1853
1854         return 0;
1855 }
1856
1857 /*
1858  * Consumer reader.
1859  */
1860 static ssize_t
1861 tracing_read_pipe(struct file *filp, char __user *ubuf,
1862                   size_t cnt, loff_t *ppos)
1863 {
1864         struct trace_iterator *iter = filp->private_data;
1865         struct trace_array_cpu *data;
1866         static cpumask_t mask;
1867         static int start;
1868         unsigned long flags;
1869         int read = 0;
1870         int cpu;
1871         int len;
1872         int ret;
1873
1874         /* return any leftover data */
1875         if (iter->seq.len > start) {
1876                 len = iter->seq.len - start;
1877                 if (cnt > len)
1878                         cnt = len;
1879                 ret = copy_to_user(ubuf, iter->seq.buffer + start, cnt);
1880                 if (ret)
1881                         cnt = -EFAULT;
1882
1883                 start += len;
1884
1885                 return cnt;
1886         }
1887
1888         trace_seq_reset(&iter->seq);
1889         start = 0;
1890
1891         while (trace_empty(iter)) {
1892                 /*
1893                  * This is a make-shift waitqueue. The reason we don't use
1894                  * an actual wait queue is because:
1895                  *  1) we only ever have one waiter
1896                  *  2) the tracing, traces all functions, we don't want
1897                  *     the overhead of calling wake_up and friends
1898                  *     (and tracing them too)
1899                  *     Anyway, this is really very primitive wakeup.
1900                  */
1901                 set_current_state(TASK_INTERRUPTIBLE);
1902                 iter->tr->waiter = current;
1903
1904                 /* sleep for one second, and try again. */
1905                 schedule_timeout(HZ);
1906
1907                 iter->tr->waiter = NULL;
1908
1909                 if (signal_pending(current))
1910                         return -EINTR;
1911
1912                 /*
1913                  * We block until we read something and tracing is disabled.
1914                  * We still block if tracing is disabled, but we have never
1915                  * read anything. This allows a user to cat this file, and
1916                  * then enable tracing. But after we have read something,
1917                  * we give an EOF when tracing is again disabled.
1918                  *
1919                  * iter->pos will be 0 if we haven't read anything.
1920                  */
1921                 if (!tracer_enabled && iter->pos)
1922                         break;
1923
1924                 continue;
1925         }
1926
1927         /* stop when tracing is finished */
1928         if (trace_empty(iter))
1929                 return 0;
1930
1931         if (cnt >= PAGE_SIZE)
1932                 cnt = PAGE_SIZE - 1;
1933
1934         memset(iter, 0, sizeof(*iter));
1935         iter->tr = &global_trace;
1936         iter->pos = -1;
1937
1938         /*
1939          * We need to stop all tracing on all CPUS to read the
1940          * the next buffer. This is a bit expensive, but is
1941          * not done often. We fill all what we can read,
1942          * and then release the locks again.
1943          */
1944
1945         cpus_clear(mask);
1946         local_irq_save(flags);
1947         for_each_possible_cpu(cpu) {
1948                 data = iter->tr->data[cpu];
1949
1950                 if (!head_page(data) || !data->trace_idx)
1951                         continue;
1952
1953                 atomic_inc(&data->disabled);
1954                 spin_lock(&data->lock);
1955                 cpu_set(cpu, mask);
1956         }
1957
1958         while (find_next_entry_inc(iter) != NULL) {
1959                 int len = iter->seq.len;
1960
1961                 ret = print_trace_line(iter);
1962                 if (!ret) {
1963                         /* don't print partial lines */
1964                         iter->seq.len = len;
1965                         break;
1966                 }
1967
1968                 trace_consume(iter);
1969
1970                 if (iter->seq.len >= cnt)
1971                         break;
1972         }
1973
1974         for_each_cpu_mask(cpu, mask) {
1975                 data = iter->tr->data[cpu];
1976                 spin_unlock(&data->lock);
1977                 atomic_dec(&data->disabled);
1978         }
1979         local_irq_restore(flags);
1980
1981         /* Now copy what we have to the user */
1982         read = iter->seq.len;
1983         if (read > cnt)
1984                 read = cnt;
1985
1986         ret = copy_to_user(ubuf, iter->seq.buffer, read);
1987
1988         if (read < iter->seq.len)
1989                 start = read;
1990         else
1991                 trace_seq_reset(&iter->seq);
1992
1993         if (ret)
1994                 read = -EFAULT;
1995
1996         return read;
1997 }
1998
1999 static struct file_operations tracing_max_lat_fops = {
2000         .open           = tracing_open_generic,
2001         .read           = tracing_max_lat_read,
2002         .write          = tracing_max_lat_write,
2003 };
2004
2005 static struct file_operations tracing_ctrl_fops = {
2006         .open           = tracing_open_generic,
2007         .read           = tracing_ctrl_read,
2008         .write          = tracing_ctrl_write,
2009 };
2010
2011 static struct file_operations set_tracer_fops = {
2012         .open           = tracing_open_generic,
2013         .read           = tracing_set_trace_read,
2014         .write          = tracing_set_trace_write,
2015 };
2016
2017 static struct file_operations tracing_pipe_fops = {
2018         .open           = tracing_open_pipe,
2019         .read           = tracing_read_pipe,
2020         .release        = tracing_release_pipe,
2021 };
2022
2023 #ifdef CONFIG_DYNAMIC_FTRACE
2024
2025 static ssize_t
2026 tracing_read_long(struct file *filp, char __user *ubuf,
2027                   size_t cnt, loff_t *ppos)
2028 {
2029         unsigned long *p = filp->private_data;
2030         char buf[64];
2031         int r;
2032
2033         r = sprintf(buf, "%ld\n", *p);
2034
2035         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2036 }
2037
2038 static struct file_operations tracing_read_long_fops = {
2039         .open           = tracing_open_generic,
2040         .read           = tracing_read_long,
2041 };
2042 #endif
2043
2044 static struct dentry *d_tracer;
2045
2046 struct dentry *tracing_init_dentry(void)
2047 {
2048         static int once;
2049
2050         if (d_tracer)
2051                 return d_tracer;
2052
2053         d_tracer = debugfs_create_dir("tracing", NULL);
2054
2055         if (!d_tracer && !once) {
2056                 once = 1;
2057                 pr_warning("Could not create debugfs directory 'tracing'\n");
2058                 return NULL;
2059         }
2060
2061         return d_tracer;
2062 }
2063
2064 #ifdef CONFIG_FTRACE_SELFTEST
2065 /* Let selftest have access to static functions in this file */
2066 #include "trace_selftest.c"
2067 #endif
2068
2069 static __init void tracer_init_debugfs(void)
2070 {
2071         struct dentry *d_tracer;
2072         struct dentry *entry;
2073
2074         d_tracer = tracing_init_dentry();
2075
2076         entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2077                                     &global_trace, &tracing_ctrl_fops);
2078         if (!entry)
2079                 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2080
2081         entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2082                                     NULL, &tracing_iter_fops);
2083         if (!entry)
2084                 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2085
2086         entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2087                                     &global_trace, &tracing_lt_fops);
2088         if (!entry)
2089                 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2090
2091         entry = debugfs_create_file("trace", 0444, d_tracer,
2092                                     &global_trace, &tracing_fops);
2093         if (!entry)
2094                 pr_warning("Could not create debugfs 'trace' entry\n");
2095
2096         entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2097                                     &global_trace, &show_traces_fops);
2098         if (!entry)
2099                 pr_warning("Could not create debugfs 'trace' entry\n");
2100
2101         entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2102                                     &global_trace, &set_tracer_fops);
2103         if (!entry)
2104                 pr_warning("Could not create debugfs 'trace' entry\n");
2105
2106         entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
2107                                     &tracing_max_latency,
2108                                     &tracing_max_lat_fops);
2109         if (!entry)
2110                 pr_warning("Could not create debugfs "
2111                            "'tracing_max_latency' entry\n");
2112
2113         entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
2114                                     &tracing_thresh, &tracing_max_lat_fops);
2115         if (!entry)
2116                 pr_warning("Could not create debugfs "
2117                            "'tracing_threash' entry\n");
2118         entry = debugfs_create_file("README", 0644, d_tracer,
2119                                     NULL, &tracing_readme_fops);
2120         if (!entry)
2121                 pr_warning("Could not create debugfs 'README' entry\n");
2122
2123         entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
2124                                     NULL, &tracing_pipe_fops);
2125         if (!entry)
2126                 pr_warning("Could not create debugfs "
2127                            "'tracing_threash' entry\n");
2128
2129 #ifdef CONFIG_DYNAMIC_FTRACE
2130         entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2131                                     &ftrace_update_tot_cnt,
2132                                     &tracing_read_long_fops);
2133         if (!entry)
2134                 pr_warning("Could not create debugfs "
2135                            "'dyn_ftrace_total_info' entry\n");
2136 #endif
2137 }
2138
2139 /* dummy trace to disable tracing */
2140 static struct tracer no_tracer __read_mostly =
2141 {
2142         .name           = "none",
2143 };
2144
2145 static int trace_alloc_page(void)
2146 {
2147         struct trace_array_cpu *data;
2148         struct page *page, *tmp;
2149         LIST_HEAD(pages);
2150         void *array;
2151         int i;
2152
2153         /* first allocate a page for each CPU */
2154         for_each_possible_cpu(i) {
2155                 array = (void *)__get_free_page(GFP_KERNEL);
2156                 if (array == NULL) {
2157                         printk(KERN_ERR "tracer: failed to allocate page"
2158                                "for trace buffer!\n");
2159                         goto free_pages;
2160                 }
2161
2162                 page = virt_to_page(array);
2163                 list_add(&page->lru, &pages);
2164
2165 /* Only allocate if we are actually using the max trace */
2166 #ifdef CONFIG_TRACER_MAX_TRACE
2167                 array = (void *)__get_free_page(GFP_KERNEL);
2168                 if (array == NULL) {
2169                         printk(KERN_ERR "tracer: failed to allocate page"
2170                                "for trace buffer!\n");
2171                         goto free_pages;
2172                 }
2173                 page = virt_to_page(array);
2174                 list_add(&page->lru, &pages);
2175 #endif
2176         }
2177
2178         /* Now that we successfully allocate a page per CPU, add them */
2179         for_each_possible_cpu(i) {
2180                 data = global_trace.data[i];
2181                 spin_lock_init(&data->lock);
2182                 lockdep_set_class(&data->lock, &data->lock_key);
2183                 page = list_entry(pages.next, struct page, lru);
2184                 list_del_init(&page->lru);
2185                 list_add_tail(&page->lru, &data->trace_pages);
2186                 ClearPageLRU(page);
2187
2188 #ifdef CONFIG_TRACER_MAX_TRACE
2189                 data = max_tr.data[i];
2190                 spin_lock_init(&data->lock);
2191                 lockdep_set_class(&data->lock, &data->lock_key);
2192                 page = list_entry(pages.next, struct page, lru);
2193                 list_del_init(&page->lru);
2194                 list_add_tail(&page->lru, &data->trace_pages);
2195                 SetPageLRU(page);
2196 #endif
2197         }
2198         global_trace.entries += ENTRIES_PER_PAGE;
2199
2200         return 0;
2201
2202  free_pages:
2203         list_for_each_entry_safe(page, tmp, &pages, lru) {
2204                 list_del_init(&page->lru);
2205                 __free_page(page);
2206         }
2207         return -ENOMEM;
2208 }
2209
2210 __init static int tracer_alloc_buffers(void)
2211 {
2212         struct trace_array_cpu *data;
2213         void *array;
2214         struct page *page;
2215         int pages = 0;
2216         int ret = -ENOMEM;
2217         int i;
2218
2219         global_trace.ctrl = tracer_enabled;
2220
2221         /* Allocate the first page for all buffers */
2222         for_each_possible_cpu(i) {
2223                 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
2224                 max_tr.data[i] = &per_cpu(max_data, i);
2225
2226                 array = (void *)__get_free_page(GFP_KERNEL);
2227                 if (array == NULL) {
2228                         printk(KERN_ERR "tracer: failed to allocate page"
2229                                "for trace buffer!\n");
2230                         goto free_buffers;
2231                 }
2232
2233                 /* set the array to the list */
2234                 INIT_LIST_HEAD(&data->trace_pages);
2235                 page = virt_to_page(array);
2236                 list_add(&page->lru, &data->trace_pages);
2237                 /* use the LRU flag to differentiate the two buffers */
2238                 ClearPageLRU(page);
2239
2240 /* Only allocate if we are actually using the max trace */
2241 #ifdef CONFIG_TRACER_MAX_TRACE
2242                 array = (void *)__get_free_page(GFP_KERNEL);
2243                 if (array == NULL) {
2244                         printk(KERN_ERR "tracer: failed to allocate page"
2245                                "for trace buffer!\n");
2246                         goto free_buffers;
2247                 }
2248
2249                 INIT_LIST_HEAD(&max_tr.data[i]->trace_pages);
2250                 page = virt_to_page(array);
2251                 list_add(&page->lru, &max_tr.data[i]->trace_pages);
2252                 SetPageLRU(page);
2253 #endif
2254         }
2255
2256         /*
2257          * Since we allocate by orders of pages, we may be able to
2258          * round up a bit.
2259          */
2260         global_trace.entries = ENTRIES_PER_PAGE;
2261         pages++;
2262
2263         while (global_trace.entries < trace_nr_entries) {
2264                 if (trace_alloc_page())
2265                         break;
2266                 pages++;
2267         }
2268         max_tr.entries = global_trace.entries;
2269
2270         pr_info("tracer: %d pages allocated for %ld",
2271                 pages, trace_nr_entries);
2272         pr_info(" entries of %ld bytes\n", (long)TRACE_ENTRY_SIZE);
2273         pr_info("   actual entries %ld\n", global_trace.entries);
2274
2275         tracer_init_debugfs();
2276
2277         trace_init_cmdlines();
2278
2279         register_tracer(&no_tracer);
2280         current_trace = &no_tracer;
2281
2282         /* All seems OK, enable tracing */
2283         tracing_disabled = 0;
2284
2285         return 0;
2286
2287  free_buffers:
2288         for (i-- ; i >= 0; i--) {
2289                 struct page *page, *tmp;
2290                 struct trace_array_cpu *data = global_trace.data[i];
2291
2292                 if (data) {
2293                         list_for_each_entry_safe(page, tmp,
2294                                                  &data->trace_pages, lru) {
2295                                 list_del_init(&page->lru);
2296                                 __free_page(page);
2297                         }
2298                 }
2299
2300 #ifdef CONFIG_TRACER_MAX_TRACE
2301                 data = max_tr.data[i];
2302                 if (data) {
2303                         list_for_each_entry_safe(page, tmp,
2304                                                  &data->trace_pages, lru) {
2305                                 list_del_init(&page->lru);
2306                                 __free_page(page);
2307                         }
2308                 }
2309 #endif
2310         }
2311         return ret;
2312 }
2313 fs_initcall(tracer_alloc_buffers);