kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/ftrace.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/slab.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31 #include <linux/rcupdate.h>
  32
  33 #include <trace/events/sched.h>
  34
  35 #include <asm/ftrace.h>
  36 #include <asm/setup.h>
  37
  38 #include "trace_output.h"
  39 #include "trace_stat.h"
  40
  41 #define FTRACE_WARN_ON(cond)                    \
  42         ({                                      \
  43                 int ___r = cond;                \
  44                 if (WARN_ON(___r))              \
  45                         ftrace_kill();          \
  46                 ___r;                           \
  47         })
  48
  49 #define FTRACE_WARN_ON_ONCE(cond)               \
  50         ({                                      \
  51                 int ___r = cond;                \
  52                 if (WARN_ON_ONCE(___r))         \
  53                         ftrace_kill();          \
  54                 ___r;                           \
  55         })
  56
  57 /* hash bits for specific function selection */
  58 #define FTRACE_HASH_BITS 7
  59 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  60 #define FTRACE_HASH_MAX_BITS 10
  61
  62 /* ftrace_enabled is a method to turn ftrace on or off */
  63 int ftrace_enabled __read_mostly;
  64 static int last_ftrace_enabled;
  65
  66 /* Quick disabling of function tracer. */
  67 int function_trace_stop;
  68
  69 /* List for set_ftrace_pid's pids. */
  70 LIST_HEAD(ftrace_pids);
  71 struct ftrace_pid {
  72         struct list_head list;
  73         struct pid *pid;
  74 };
  75
  76 /*
  77  * ftrace_disabled is set when an anomaly is discovered.
  78  * ftrace_disabled is much stronger than ftrace_enabled.
  79  */
  80 static int ftrace_disabled __read_mostly;
  81
  82 static DEFINE_MUTEX(ftrace_lock);
  83
  84 static struct ftrace_ops ftrace_list_end __read_mostly =
  85 {
  86         .func           = ftrace_stub,
  87 };
  88
  89 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  90 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  91 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  92 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  93
  94 /*
  95  * Traverse the ftrace_list, invoking all entries.  The reason that we
  96  * can use rcu_dereference_raw() is that elements removed from this list
  97  * are simply leaked, so there is no need to interact with a grace-period
  98  * mechanism.  The rcu_dereference_raw() calls are needed to handle
  99  * concurrent insertions into the ftrace_list.
 100  *
 101  * Silly Alpha and silly pointer-speculation compiler optimizations!
 102  */
 103 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
 104 {
 105         struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/
 106
 107         while (op != &ftrace_list_end) {
 108                 op->func(ip, parent_ip);
 109                 op = rcu_dereference_raw(op->next); /*see above*/
 110         };
 111 }
 112
 113 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
 114 {
 115         if (!test_tsk_trace_trace(current))
 116                 return;
 117
 118         ftrace_pid_function(ip, parent_ip);
 119 }
 120
 121 static void set_ftrace_pid_function(ftrace_func_t func)
 122 {
 123         /* do not set ftrace_pid_function to itself! */
 124         if (func != ftrace_pid_func)
 125                 ftrace_pid_function = func;
 126 }
 127
 128 /**
 129  * clear_ftrace_function - reset the ftrace function
 130  *
 131  * This NULLs the ftrace function and in essence stops
 132  * tracing.  There may be lag
 133  */
 134 void clear_ftrace_function(void)
 135 {
 136         ftrace_trace_function = ftrace_stub;
 137         __ftrace_trace_function = ftrace_stub;
 138         ftrace_pid_function = ftrace_stub;
 139 }
 140
 141 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 142 /*
 143  * For those archs that do not test ftrace_trace_stop in their
 144  * mcount call site, we need to do it from C.
 145  */
 146 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 147 {
 148         if (function_trace_stop)
 149                 return;
 150
 151         __ftrace_trace_function(ip, parent_ip);
 152 }
 153 #endif
 154
 155 static void update_ftrace_function(void)
 156 {
 157         ftrace_func_t func;
 158
 159         /*
 160          * If there's only one function registered, then call that
 161          * function directly. Otherwise, we need to iterate over the
 162          * registered callers.
 163          */
 164         if (ftrace_list == &ftrace_list_end ||
 165             ftrace_list->next == &ftrace_list_end)
 166                 func = ftrace_list->func;
 167         else
 168                 func = ftrace_list_func;
 169
 170         /* If we filter on pids, update to use the pid function */
 171         if (!list_empty(&ftrace_pids)) {
 172                 set_ftrace_pid_function(func);
 173                 func = ftrace_pid_func;
 174         }
 175 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 176         ftrace_trace_function = func;
 177 #else
 178         __ftrace_trace_function = func;
 179         ftrace_trace_function = ftrace_test_stop_func;
 180 #endif
 181 }
 182
 183 static int __register_ftrace_function(struct ftrace_ops *ops)
 184 {
 185         ops->next = ftrace_list;
 186         /*
 187          * We are entering ops into the ftrace_list but another
 188          * CPU might be walking that list. We need to make sure
 189          * the ops->next pointer is valid before another CPU sees
 190          * the ops pointer included into the ftrace_list.
 191          */
 192         rcu_assign_pointer(ftrace_list, ops);
 193
 194         if (ftrace_enabled)
 195                 update_ftrace_function();
 196
 197         return 0;
 198 }
 199
 200 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 201 {
 202         struct ftrace_ops **p;
 203
 204         /*
 205          * If we are removing the last function, then simply point
 206          * to the ftrace_stub.
 207          */
 208         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 209                 ftrace_trace_function = ftrace_stub;
 210                 ftrace_list = &ftrace_list_end;
 211                 return 0;
 212         }
 213
 214         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 215                 if (*p == ops)
 216                         break;
 217
 218         if (*p != ops)
 219                 return -1;
 220
 221         *p = (*p)->next;
 222
 223         if (ftrace_enabled)
 224                 update_ftrace_function();
 225
 226         return 0;
 227 }
 228
 229 static void ftrace_update_pid_func(void)
 230 {
 231         /* Only do something if we are tracing something */
 232         if (ftrace_trace_function == ftrace_stub)
 233                 return;
 234
 235         update_ftrace_function();
 236 }
 237
 238 #ifdef CONFIG_FUNCTION_PROFILER
 239 struct ftrace_profile {
 240         struct hlist_node               node;
 241         unsigned long                   ip;
 242         unsigned long                   counter;
 243 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 244         unsigned long long              time;
 245         unsigned long long              time_squared;
 246 #endif
 247 };
 248
 249 struct ftrace_profile_page {
 250         struct ftrace_profile_page      *next;
 251         unsigned long                   index;
 252         struct ftrace_profile           records[];
 253 };
 254
 255 struct ftrace_profile_stat {
 256         atomic_t                        disabled;
 257         struct hlist_head               *hash;
 258         struct ftrace_profile_page      *pages;
 259         struct ftrace_profile_page      *start;
 260         struct tracer_stat              stat;
 261 };
 262
 263 #define PROFILE_RECORDS_SIZE                                            \
 264         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 265
 266 #define PROFILES_PER_PAGE                                       \
 267         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 268
 269 static int ftrace_profile_bits __read_mostly;
 270 static int ftrace_profile_enabled __read_mostly;
 271
 272 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 273 static DEFINE_MUTEX(ftrace_profile_lock);
 274
 275 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 276
 277 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 278
 279 static void *
 280 function_stat_next(void *v, int idx)
 281 {
 282         struct ftrace_profile *rec = v;
 283         struct ftrace_profile_page *pg;
 284
 285         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 286
 287  again:
 288         if (idx != 0)
 289                 rec++;
 290
 291         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 292                 pg = pg->next;
 293                 if (!pg)
 294                         return NULL;
 295                 rec = &pg->records[0];
 296                 if (!rec->counter)
 297                         goto again;
 298         }
 299
 300         return rec;
 301 }
 302
 303 static void *function_stat_start(struct tracer_stat *trace)
 304 {
 305         struct ftrace_profile_stat *stat =
 306                 container_of(trace, struct ftrace_profile_stat, stat);
 307
 308         if (!stat || !stat->start)
 309                 return NULL;
 310
 311         return function_stat_next(&stat->start->records[0], 0);
 312 }
 313
 314 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 315 /* function graph compares on total time */
 316 static int function_stat_cmp(void *p1, void *p2)
 317 {
 318         struct ftrace_profile *a = p1;
 319         struct ftrace_profile *b = p2;
 320
 321         if (a->time < b->time)
 322                 return -1;
 323         if (a->time > b->time)
 324                 return 1;
 325         else
 326                 return 0;
 327 }
 328 #else
 329 /* not function graph compares against hits */
 330 static int function_stat_cmp(void *p1, void *p2)
 331 {
 332         struct ftrace_profile *a = p1;
 333         struct ftrace_profile *b = p2;
 334
 335         if (a->counter < b->counter)
 336                 return -1;
 337         if (a->counter > b->counter)
 338                 return 1;
 339         else
 340                 return 0;
 341 }
 342 #endif
 343
 344 static int function_stat_headers(struct seq_file *m)
 345 {
 346 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 347         seq_printf(m, "  Function                               "
 348                    "Hit    Time            Avg             s^2\n"
 349                       "  --------                               "
 350                    "---    ----            ---             ---\n");
 351 #else
 352         seq_printf(m, "  Function                               Hit\n"
 353                       "  --------                               ---\n");
 354 #endif
 355         return 0;
 356 }
 357
 358 static int function_stat_show(struct seq_file *m, void *v)
 359 {
 360         struct ftrace_profile *rec = v;
 361         char str[KSYM_SYMBOL_LEN];
 362         int ret = 0;
 363 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 364         static struct trace_seq s;
 365         unsigned long long avg;
 366         unsigned long long stddev;
 367 #endif
 368         mutex_lock(&ftrace_profile_lock);
 369
 370         /* we raced with function_profile_reset() */
 371         if (unlikely(rec->counter == 0)) {
 372                 ret = -EBUSY;
 373                 goto out;
 374         }
 375
 376         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 377         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 378
 379 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 380         seq_printf(m, "    ");
 381         avg = rec->time;
 382         do_div(avg, rec->counter);
 383
 384         /* Sample standard deviation (s^2) */
 385         if (rec->counter <= 1)
 386                 stddev = 0;
 387         else {
 388                 stddev = rec->time_squared - rec->counter * avg * avg;
 389                 /*
 390                  * Divide only 1000 for ns^2 -> us^2 conversion.
 391                  * trace_print_graph_duration will divide 1000 again.
 392                  */
 393                 do_div(stddev, (rec->counter - 1) * 1000);
 394         }
 395
 396         trace_seq_init(&s);
 397         trace_print_graph_duration(rec->time, &s);
 398         trace_seq_puts(&s, "    ");
 399         trace_print_graph_duration(avg, &s);
 400         trace_seq_puts(&s, "    ");
 401         trace_print_graph_duration(stddev, &s);
 402         trace_print_seq(m, &s);
 403 #endif
 404         seq_putc(m, '\n');
 405 out:
 406         mutex_unlock(&ftrace_profile_lock);
 407
 408         return ret;
 409 }
 410
 411 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 412 {
 413         struct ftrace_profile_page *pg;
 414
 415         pg = stat->pages = stat->start;
 416
 417         while (pg) {
 418                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 419                 pg->index = 0;
 420                 pg = pg->next;
 421         }
 422
 423         memset(stat->hash, 0,
 424                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 425 }
 426
 427 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 428 {
 429         struct ftrace_profile_page *pg;
 430         int functions;
 431         int pages;
 432         int i;
 433
 434         /* If we already allocated, do nothing */
 435         if (stat->pages)
 436                 return 0;
 437
 438         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 439         if (!stat->pages)
 440                 return -ENOMEM;
 441
 442 #ifdef CONFIG_DYNAMIC_FTRACE
 443         functions = ftrace_update_tot_cnt;
 444 #else
 445         /*
 446          * We do not know the number of functions that exist because
 447          * dynamic tracing is what counts them. With past experience
 448          * we have around 20K functions. That should be more than enough.
 449          * It is highly unlikely we will execute every function in
 450          * the kernel.
 451          */
 452         functions = 20000;
 453 #endif
 454
 455         pg = stat->start = stat->pages;
 456
 457         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 458
 459         for (i = 0; i < pages; i++) {
 460                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 461                 if (!pg->next)
 462                         goto out_free;
 463                 pg = pg->next;
 464         }
 465
 466         return 0;
 467
 468  out_free:
 469         pg = stat->start;
 470         while (pg) {
 471                 unsigned long tmp = (unsigned long)pg;
 472
 473                 pg = pg->next;
 474                 free_page(tmp);
 475         }
 476
 477         free_page((unsigned long)stat->pages);
 478         stat->pages = NULL;
 479         stat->start = NULL;
 480
 481         return -ENOMEM;
 482 }
 483
 484 static int ftrace_profile_init_cpu(int cpu)
 485 {
 486         struct ftrace_profile_stat *stat;
 487         int size;
 488
 489         stat = &per_cpu(ftrace_profile_stats, cpu);
 490
 491         if (stat->hash) {
 492                 /* If the profile is already created, simply reset it */
 493                 ftrace_profile_reset(stat);
 494                 return 0;
 495         }
 496
 497         /*
 498          * We are profiling all functions, but usually only a few thousand
 499          * functions are hit. We'll make a hash of 1024 items.
 500          */
 501         size = FTRACE_PROFILE_HASH_SIZE;
 502
 503         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 504
 505         if (!stat->hash)
 506                 return -ENOMEM;
 507
 508         if (!ftrace_profile_bits) {
 509                 size--;
 510
 511                 for (; size; size >>= 1)
 512                         ftrace_profile_bits++;
 513         }
 514
 515         /* Preallocate the function profiling pages */
 516         if (ftrace_profile_pages_init(stat) < 0) {
 517                 kfree(stat->hash);
 518                 stat->hash = NULL;
 519                 return -ENOMEM;
 520         }
 521
 522         return 0;
 523 }
 524
 525 static int ftrace_profile_init(void)
 526 {
 527         int cpu;
 528         int ret = 0;
 529
 530         for_each_online_cpu(cpu) {
 531                 ret = ftrace_profile_init_cpu(cpu);
 532                 if (ret)
 533                         break;
 534         }
 535
 536         return ret;
 537 }
 538
 539 /* interrupts must be disabled */
 540 static struct ftrace_profile *
 541 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 542 {
 543         struct ftrace_profile *rec;
 544         struct hlist_head *hhd;
 545         struct hlist_node *n;
 546         unsigned long key;
 547
 548         key = hash_long(ip, ftrace_profile_bits);
 549         hhd = &stat->hash[key];
 550
 551         if (hlist_empty(hhd))
 552                 return NULL;
 553
 554         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 555                 if (rec->ip == ip)
 556                         return rec;
 557         }
 558
 559         return NULL;
 560 }
 561
 562 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 563                                struct ftrace_profile *rec)
 564 {
 565         unsigned long key;
 566
 567         key = hash_long(rec->ip, ftrace_profile_bits);
 568         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 569 }
 570
 571 /*
 572  * The memory is already allocated, this simply finds a new record to use.
 573  */
 574 static struct ftrace_profile *
 575 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 576 {
 577         struct ftrace_profile *rec = NULL;
 578
 579         /* prevent recursion (from NMIs) */
 580         if (atomic_inc_return(&stat->disabled) != 1)
 581                 goto out;
 582
 583         /*
 584          * Try to find the function again since an NMI
 585          * could have added it
 586          */
 587         rec = ftrace_find_profiled_func(stat, ip);
 588         if (rec)
 589                 goto out;
 590
 591         if (stat->pages->index == PROFILES_PER_PAGE) {
 592                 if (!stat->pages->next)
 593                         goto out;
 594                 stat->pages = stat->pages->next;
 595         }
 596
 597         rec = &stat->pages->records[stat->pages->index++];
 598         rec->ip = ip;
 599         ftrace_add_profile(stat, rec);
 600
 601  out:
 602         atomic_dec(&stat->disabled);
 603
 604         return rec;
 605 }
 606
 607 static void
 608 function_profile_call(unsigned long ip, unsigned long parent_ip)
 609 {
 610         struct ftrace_profile_stat *stat;
 611         struct ftrace_profile *rec;
 612         unsigned long flags;
 613
 614         if (!ftrace_profile_enabled)
 615                 return;
 616
 617         local_irq_save(flags);
 618
 619         stat = &__get_cpu_var(ftrace_profile_stats);
 620         if (!stat->hash || !ftrace_profile_enabled)
 621                 goto out;
 622
 623         rec = ftrace_find_profiled_func(stat, ip);
 624         if (!rec) {
 625                 rec = ftrace_profile_alloc(stat, ip);
 626                 if (!rec)
 627                         goto out;
 628         }
 629
 630         rec->counter++;
 631  out:
 632         local_irq_restore(flags);
 633 }
 634
 635 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 636 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 637 {
 638         function_profile_call(trace->func, 0);
 639         return 1;
 640 }
 641
 642 static void profile_graph_return(struct ftrace_graph_ret *trace)
 643 {
 644         struct ftrace_profile_stat *stat;
 645         unsigned long long calltime;
 646         struct ftrace_profile *rec;
 647         unsigned long flags;
 648
 649         local_irq_save(flags);
 650         stat = &__get_cpu_var(ftrace_profile_stats);
 651         if (!stat->hash || !ftrace_profile_enabled)
 652                 goto out;
 653
 654         /* If the calltime was zero'd ignore it */
 655         if (!trace->calltime)
 656                 goto out;
 657
 658         calltime = trace->rettime - trace->calltime;
 659
 660         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 661                 int index;
 662
 663                 index = trace->depth;
 664
 665                 /* Append this call time to the parent time to subtract */
 666                 if (index)
 667                         current->ret_stack[index - 1].subtime += calltime;
 668
 669                 if (current->ret_stack[index].subtime < calltime)
 670                         calltime -= current->ret_stack[index].subtime;
 671                 else
 672                         calltime = 0;
 673         }
 674
 675         rec = ftrace_find_profiled_func(stat, trace->func);
 676         if (rec) {
 677                 rec->time += calltime;
 678                 rec->time_squared += calltime * calltime;
 679         }
 680
 681  out:
 682         local_irq_restore(flags);
 683 }
 684
 685 static int register_ftrace_profiler(void)
 686 {
 687         return register_ftrace_graph(&profile_graph_return,
 688                                      &profile_graph_entry);
 689 }
 690
 691 static void unregister_ftrace_profiler(void)
 692 {
 693         unregister_ftrace_graph();
 694 }
 695 #else
 696 static struct ftrace_ops ftrace_profile_ops __read_mostly =
 697 {
 698         .func           = function_profile_call,
 699 };
 700
 701 static int register_ftrace_profiler(void)
 702 {
 703         return register_ftrace_function(&ftrace_profile_ops);
 704 }
 705
 706 static void unregister_ftrace_profiler(void)
 707 {
 708         unregister_ftrace_function(&ftrace_profile_ops);
 709 }
 710 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 711
 712 static ssize_t
 713 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 714                      size_t cnt, loff_t *ppos)
 715 {
 716         unsigned long val;
 717         char buf[64];           /* big enough to hold a number */
 718         int ret;
 719
 720         if (cnt >= sizeof(buf))
 721                 return -EINVAL;
 722
 723         if (copy_from_user(&buf, ubuf, cnt))
 724                 return -EFAULT;
 725
 726         buf[cnt] = 0;
 727
 728         ret = strict_strtoul(buf, 10, &val);
 729         if (ret < 0)
 730                 return ret;
 731
 732         val = !!val;
 733
 734         mutex_lock(&ftrace_profile_lock);
 735         if (ftrace_profile_enabled ^ val) {
 736                 if (val) {
 737                         ret = ftrace_profile_init();
 738                         if (ret < 0) {
 739                                 cnt = ret;
 740                                 goto out;
 741                         }
 742
 743                         ret = register_ftrace_profiler();
 744                         if (ret < 0) {
 745                                 cnt = ret;
 746                                 goto out;
 747                         }
 748                         ftrace_profile_enabled = 1;
 749                 } else {
 750                         ftrace_profile_enabled = 0;
 751                         /*
 752                          * unregister_ftrace_profiler calls stop_machine
 753                          * so this acts like an synchronize_sched.
 754                          */
 755                         unregister_ftrace_profiler();
 756                 }
 757         }
 758  out:
 759         mutex_unlock(&ftrace_profile_lock);
 760
 761         *ppos += cnt;
 762
 763         return cnt;
 764 }
 765
 766 static ssize_t
 767 ftrace_profile_read(struct file *filp, char __user *ubuf,
 768                      size_t cnt, loff_t *ppos)
 769 {
 770         char buf[64];           /* big enough to hold a number */
 771         int r;
 772
 773         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 774         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 775 }
 776
 777 static const struct file_operations ftrace_profile_fops = {
 778         .open           = tracing_open_generic,
 779         .read           = ftrace_profile_read,
 780         .write          = ftrace_profile_write,
 781         .llseek         = default_llseek,
 782 };
 783
 784 /* used to initialize the real stat files */
 785 static struct tracer_stat function_stats __initdata = {
 786         .name           = "functions",
 787         .stat_start     = function_stat_start,
 788         .stat_next      = function_stat_next,
 789         .stat_cmp       = function_stat_cmp,
 790         .stat_headers   = function_stat_headers,
 791         .stat_show      = function_stat_show
 792 };
 793
 794 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 795 {
 796         struct ftrace_profile_stat *stat;
 797         struct dentry *entry;
 798         char *name;
 799         int ret;
 800         int cpu;
 801
 802         for_each_possible_cpu(cpu) {
 803                 stat = &per_cpu(ftrace_profile_stats, cpu);
 804
 805                 /* allocate enough for function name + cpu number */
 806                 name = kmalloc(32, GFP_KERNEL);
 807                 if (!name) {
 808                         /*
 809                          * The files created are permanent, if something happens
 810                          * we still do not free memory.
 811                          */
 812                         WARN(1,
 813                              "Could not allocate stat file for cpu %d\n",
 814                              cpu);
 815                         return;
 816                 }
 817                 stat->stat = function_stats;
 818                 snprintf(name, 32, "function%d", cpu);
 819                 stat->stat.name = name;
 820                 ret = register_stat_tracer(&stat->stat);
 821                 if (ret) {
 822                         WARN(1,
 823                              "Could not register function stat for cpu %d\n",
 824                              cpu);
 825                         kfree(name);
 826                         return;
 827                 }
 828         }
 829
 830         entry = debugfs_create_file("function_profile_enabled", 0644,
 831                                     d_tracer, NULL, &ftrace_profile_fops);
 832         if (!entry)
 833                 pr_warning("Could not create debugfs "
 834                            "'function_profile_enabled' entry\n");
 835 }
 836
 837 #else /* CONFIG_FUNCTION_PROFILER */
 838 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 839 {
 840 }
 841 #endif /* CONFIG_FUNCTION_PROFILER */
 842
 843 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 844
 845 #ifdef CONFIG_DYNAMIC_FTRACE
 846
 847 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 848 # error Dynamic ftrace depends on MCOUNT_RECORD
 849 #endif
 850
 851 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 852
 853 struct ftrace_func_probe {
 854         struct hlist_node       node;
 855         struct ftrace_probe_ops *ops;
 856         unsigned long           flags;
 857         unsigned long           ip;
 858         void                    *data;
 859         struct rcu_head         rcu;
 860 };
 861
 862 enum {
 863         FTRACE_ENABLE_CALLS             = (1 << 0),
 864         FTRACE_DISABLE_CALLS            = (1 << 1),
 865         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 866         FTRACE_START_FUNC_RET           = (1 << 3),
 867         FTRACE_STOP_FUNC_RET            = (1 << 4),
 868 };
 869 struct ftrace_func_entry {
 870         struct hlist_node hlist;
 871         unsigned long ip;
 872 };
 873
 874 struct ftrace_hash {
 875         unsigned long           size_bits;
 876         struct hlist_head       *buckets;
 877         unsigned long           count;
 878 };
 879
 880 static struct hlist_head notrace_buckets[1 << FTRACE_HASH_MAX_BITS];
 881 static struct ftrace_hash notrace_hash = {
 882         .size_bits = FTRACE_HASH_MAX_BITS,
 883         .buckets = notrace_buckets,
 884 };
 885
 886 static int ftrace_filtered;
 887
 888 static struct dyn_ftrace *ftrace_new_addrs;
 889
 890 static DEFINE_MUTEX(ftrace_regex_lock);
 891
 892 struct ftrace_page {
 893         struct ftrace_page      *next;
 894         int                     index;
 895         struct dyn_ftrace       records[];
 896 };
 897
 898 #define ENTRIES_PER_PAGE \
 899   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 900
 901 /* estimate from running different kernels */
 902 #define NR_TO_INIT              10000
 903
 904 static struct ftrace_page       *ftrace_pages_start;
 905 static struct ftrace_page       *ftrace_pages;
 906
 907 static struct dyn_ftrace *ftrace_free_records;
 908
 909 static struct ftrace_func_entry *
 910 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
 911 {
 912         unsigned long key;
 913         struct ftrace_func_entry *entry;
 914         struct hlist_head *hhd;
 915         struct hlist_node *n;
 916
 917         if (!hash->count)
 918                 return NULL;
 919
 920         if (hash->size_bits > 0)
 921                 key = hash_long(ip, hash->size_bits);
 922         else
 923                 key = 0;
 924
 925         hhd = &hash->buckets[key];
 926
 927         hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
 928                 if (entry->ip == ip)
 929                         return entry;
 930         }
 931         return NULL;
 932 }
 933
 934 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
 935 {
 936         struct ftrace_func_entry *entry;
 937         struct hlist_head *hhd;
 938         unsigned long key;
 939
 940         entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 941         if (!entry)
 942                 return -ENOMEM;
 943
 944         if (hash->size_bits)
 945                 key = hash_long(ip, hash->size_bits);
 946         else
 947                 key = 0;
 948
 949         entry->ip = ip;
 950         hhd = &hash->buckets[key];
 951         hlist_add_head(&entry->hlist, hhd);
 952         hash->count++;
 953
 954         return 0;
 955 }
 956
 957 static void
 958 remove_hash_entry(struct ftrace_hash *hash,
 959                   struct ftrace_func_entry *entry)
 960 {
 961         hlist_del(&entry->hlist);
 962         kfree(entry);
 963         hash->count--;
 964 }
 965
 966 static void ftrace_hash_clear(struct ftrace_hash *hash)
 967 {
 968         struct hlist_head *hhd;
 969         struct hlist_node *tp, *tn;
 970         struct ftrace_func_entry *entry;
 971         int size = 1 << hash->size_bits;
 972         int i;
 973
 974         for (i = 0; i < size; i++) {
 975                 hhd = &hash->buckets[i];
 976                 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
 977                         remove_hash_entry(hash, entry);
 978         }
 979         FTRACE_WARN_ON(hash->count);
 980 }
 981
 982 /*
 983  * This is a double for. Do not use 'break' to break out of the loop,
 984  * you must use a goto.
 985  */
 986 #define do_for_each_ftrace_rec(pg, rec)                                 \
 987         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 988                 int _____i;                                             \
 989                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 990                         rec = &pg->records[_____i];
 991
 992 #define while_for_each_ftrace_rec()             \
 993                 }                               \
 994         }
 995
 996 static void ftrace_free_rec(struct dyn_ftrace *rec)
 997 {
 998         rec->freelist = ftrace_free_records;
 999         ftrace_free_records = rec;
1000         rec->flags |= FTRACE_FL_FREE;
1001 }
1002
1003 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
1004 {
1005         struct dyn_ftrace *rec;
1006
1007         /* First check for freed records */
1008         if (ftrace_free_records) {
1009                 rec = ftrace_free_records;
1010
1011                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
1012                         FTRACE_WARN_ON_ONCE(1);
1013                         ftrace_free_records = NULL;
1014                         return NULL;
1015                 }
1016
1017                 ftrace_free_records = rec->freelist;
1018                 memset(rec, 0, sizeof(*rec));
1019                 return rec;
1020         }
1021
1022         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
1023                 if (!ftrace_pages->next) {
1024                         /* allocate another page */
1025                         ftrace_pages->next =
1026                                 (void *)get_zeroed_page(GFP_KERNEL);
1027                         if (!ftrace_pages->next)
1028                                 return NULL;
1029                 }
1030                 ftrace_pages = ftrace_pages->next;
1031         }
1032
1033         return &ftrace_pages->records[ftrace_pages->index++];
1034 }
1035
1036 static struct dyn_ftrace *
1037 ftrace_record_ip(unsigned long ip)
1038 {
1039         struct dyn_ftrace *rec;
1040
1041         if (ftrace_disabled)
1042                 return NULL;
1043
1044         rec = ftrace_alloc_dyn_node(ip);
1045         if (!rec)
1046                 return NULL;
1047
1048         rec->ip = ip;
1049         rec->newlist = ftrace_new_addrs;
1050         ftrace_new_addrs = rec;
1051
1052         return rec;
1053 }
1054
1055 static void print_ip_ins(const char *fmt, unsigned char *p)
1056 {
1057         int i;
1058
1059         printk(KERN_CONT "%s", fmt);
1060
1061         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1062                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1063 }
1064
1065 static void ftrace_bug(int failed, unsigned long ip)
1066 {
1067         switch (failed) {
1068         case -EFAULT:
1069                 FTRACE_WARN_ON_ONCE(1);
1070                 pr_info("ftrace faulted on modifying ");
1071                 print_ip_sym(ip);
1072                 break;
1073         case -EINVAL:
1074                 FTRACE_WARN_ON_ONCE(1);
1075                 pr_info("ftrace failed to modify ");
1076                 print_ip_sym(ip);
1077                 print_ip_ins(" actual: ", (unsigned char *)ip);
1078                 printk(KERN_CONT "\n");
1079                 break;
1080         case -EPERM:
1081                 FTRACE_WARN_ON_ONCE(1);
1082                 pr_info("ftrace faulted on writing ");
1083                 print_ip_sym(ip);
1084                 break;
1085         default:
1086                 FTRACE_WARN_ON_ONCE(1);
1087                 pr_info("ftrace faulted on unknown error ");
1088                 print_ip_sym(ip);
1089         }
1090 }
1091
1092
1093 /* Return 1 if the address range is reserved for ftrace */
1094 int ftrace_text_reserved(void *start, void *end)
1095 {
1096         struct dyn_ftrace *rec;
1097         struct ftrace_page *pg;
1098
1099         do_for_each_ftrace_rec(pg, rec) {
1100                 if (rec->ip <= (unsigned long)end &&
1101                     rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1102                         return 1;
1103         } while_for_each_ftrace_rec();
1104         return 0;
1105 }
1106
1107
1108 static int
1109 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1110 {
1111         unsigned long ftrace_addr;
1112         unsigned long flag = 0UL;
1113
1114         ftrace_addr = (unsigned long)FTRACE_ADDR;
1115
1116         /*
1117          * If this record is not to be traced or we want to disable it,
1118          * then disable it.
1119          *
1120          * If we want to enable it and filtering is off, then enable it.
1121          *
1122          * If we want to enable it and filtering is on, enable it only if
1123          * it's filtered
1124          */
1125         if (enable && !ftrace_lookup_ip(&notrace_hash, rec->ip)) {
1126                 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1127                         flag = FTRACE_FL_ENABLED;
1128         }
1129
1130         /* If the state of this record hasn't changed, then do nothing */
1131         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1132                 return 0;
1133
1134         if (flag) {
1135                 rec->flags |= FTRACE_FL_ENABLED;
1136                 return ftrace_make_call(rec, ftrace_addr);
1137         }
1138
1139         rec->flags &= ~FTRACE_FL_ENABLED;
1140         return ftrace_make_nop(NULL, rec, ftrace_addr);
1141 }
1142
1143 static void ftrace_replace_code(int enable)
1144 {
1145         struct dyn_ftrace *rec;
1146         struct ftrace_page *pg;
1147         int failed;
1148
1149         if (unlikely(ftrace_disabled))
1150                 return;
1151
1152         do_for_each_ftrace_rec(pg, rec) {
1153                 /* Skip over free records */
1154                 if (rec->flags & FTRACE_FL_FREE)
1155                         continue;
1156
1157                 failed = __ftrace_replace_code(rec, enable);
1158                 if (failed) {
1159                         ftrace_bug(failed, rec->ip);
1160                         /* Stop processing */
1161                         return;
1162                 }
1163         } while_for_each_ftrace_rec();
1164 }
1165
1166 static int
1167 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1168 {
1169         unsigned long ip;
1170         int ret;
1171
1172         ip = rec->ip;
1173
1174         if (unlikely(ftrace_disabled))
1175                 return 0;
1176
1177         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1178         if (ret) {
1179                 ftrace_bug(ret, ip);
1180                 return 0;
1181         }
1182         return 1;
1183 }
1184
1185 /*
1186  * archs can override this function if they must do something
1187  * before the modifying code is performed.
1188  */
1189 int __weak ftrace_arch_code_modify_prepare(void)
1190 {
1191         return 0;
1192 }
1193
1194 /*
1195  * archs can override this function if they must do something
1196  * after the modifying code is performed.
1197  */
1198 int __weak ftrace_arch_code_modify_post_process(void)
1199 {
1200         return 0;
1201 }
1202
1203 static int __ftrace_modify_code(void *data)
1204 {
1205         int *command = data;
1206
1207         if (*command & FTRACE_ENABLE_CALLS)
1208                 ftrace_replace_code(1);
1209         else if (*command & FTRACE_DISABLE_CALLS)
1210                 ftrace_replace_code(0);
1211
1212         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1213                 ftrace_update_ftrace_func(ftrace_trace_function);
1214
1215         if (*command & FTRACE_START_FUNC_RET)
1216                 ftrace_enable_ftrace_graph_caller();
1217         else if (*command & FTRACE_STOP_FUNC_RET)
1218                 ftrace_disable_ftrace_graph_caller();
1219
1220         return 0;
1221 }
1222
1223 static void ftrace_run_update_code(int command)
1224 {
1225         int ret;
1226
1227         ret = ftrace_arch_code_modify_prepare();
1228         FTRACE_WARN_ON(ret);
1229         if (ret)
1230                 return;
1231
1232         stop_machine(__ftrace_modify_code, &command, NULL);
1233
1234         ret = ftrace_arch_code_modify_post_process();
1235         FTRACE_WARN_ON(ret);
1236 }
1237
1238 static ftrace_func_t saved_ftrace_func;
1239 static int ftrace_start_up;
1240
1241 static void ftrace_startup_enable(int command)
1242 {
1243         if (saved_ftrace_func != ftrace_trace_function) {
1244                 saved_ftrace_func = ftrace_trace_function;
1245                 command |= FTRACE_UPDATE_TRACE_FUNC;
1246         }
1247
1248         if (!command || !ftrace_enabled)
1249                 return;
1250
1251         ftrace_run_update_code(command);
1252 }
1253
1254 static void ftrace_startup(int command)
1255 {
1256         if (unlikely(ftrace_disabled))
1257                 return;
1258
1259         ftrace_start_up++;
1260         command |= FTRACE_ENABLE_CALLS;
1261
1262         ftrace_startup_enable(command);
1263 }
1264
1265 static void ftrace_shutdown(int command)
1266 {
1267         if (unlikely(ftrace_disabled))
1268                 return;
1269
1270         ftrace_start_up--;
1271         /*
1272          * Just warn in case of unbalance, no need to kill ftrace, it's not
1273          * critical but the ftrace_call callers may be never nopped again after
1274          * further ftrace uses.
1275          */
1276         WARN_ON_ONCE(ftrace_start_up < 0);
1277
1278         if (!ftrace_start_up)
1279                 command |= FTRACE_DISABLE_CALLS;
1280
1281         if (saved_ftrace_func != ftrace_trace_function) {
1282                 saved_ftrace_func = ftrace_trace_function;
1283                 command |= FTRACE_UPDATE_TRACE_FUNC;
1284         }
1285
1286         if (!command || !ftrace_enabled)
1287                 return;
1288
1289         ftrace_run_update_code(command);
1290 }
1291
1292 static void ftrace_startup_sysctl(void)
1293 {
1294         if (unlikely(ftrace_disabled))
1295                 return;
1296
1297         /* Force update next time */
1298         saved_ftrace_func = NULL;
1299         /* ftrace_start_up is true if we want ftrace running */
1300         if (ftrace_start_up)
1301                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1302 }
1303
1304 static void ftrace_shutdown_sysctl(void)
1305 {
1306         if (unlikely(ftrace_disabled))
1307                 return;
1308
1309         /* ftrace_start_up is true if ftrace is running */
1310         if (ftrace_start_up)
1311                 ftrace_run_update_code(FTRACE_DISABLE_CALLS);
1312 }
1313
1314 static cycle_t          ftrace_update_time;
1315 static unsigned long    ftrace_update_cnt;
1316 unsigned long           ftrace_update_tot_cnt;
1317
1318 static int ftrace_update_code(struct module *mod)
1319 {
1320         struct dyn_ftrace *p;
1321         cycle_t start, stop;
1322
1323         start = ftrace_now(raw_smp_processor_id());
1324         ftrace_update_cnt = 0;
1325
1326         while (ftrace_new_addrs) {
1327
1328                 /* If something went wrong, bail without enabling anything */
1329                 if (unlikely(ftrace_disabled))
1330                         return -1;
1331
1332                 p = ftrace_new_addrs;
1333                 ftrace_new_addrs = p->newlist;
1334                 p->flags = 0L;
1335
1336                 /*
1337                  * Do the initial record conversion from mcount jump
1338                  * to the NOP instructions.
1339                  */
1340                 if (!ftrace_code_disable(mod, p)) {
1341                         ftrace_free_rec(p);
1342                         /* Game over */
1343                         break;
1344                 }
1345
1346                 ftrace_update_cnt++;
1347
1348                 /*
1349                  * If the tracing is enabled, go ahead and enable the record.
1350                  *
1351                  * The reason not to enable the record immediatelly is the
1352                  * inherent check of ftrace_make_nop/ftrace_make_call for
1353                  * correct previous instructions.  Making first the NOP
1354                  * conversion puts the module to the correct state, thus
1355                  * passing the ftrace_make_call check.
1356                  */
1357                 if (ftrace_start_up) {
1358                         int failed = __ftrace_replace_code(p, 1);
1359                         if (failed) {
1360                                 ftrace_bug(failed, p->ip);
1361                                 ftrace_free_rec(p);
1362                         }
1363                 }
1364         }
1365
1366         stop = ftrace_now(raw_smp_processor_id());
1367         ftrace_update_time = stop - start;
1368         ftrace_update_tot_cnt += ftrace_update_cnt;
1369
1370         return 0;
1371 }
1372
1373 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1374 {
1375         struct ftrace_page *pg;
1376         int cnt;
1377         int i;
1378
1379         /* allocate a few pages */
1380         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1381         if (!ftrace_pages_start)
1382                 return -1;
1383
1384         /*
1385          * Allocate a few more pages.
1386          *
1387          * TODO: have some parser search vmlinux before
1388          *   final linking to find all calls to ftrace.
1389          *   Then we can:
1390          *    a) know how many pages to allocate.
1391          *     and/or
1392          *    b) set up the table then.
1393          *
1394          *  The dynamic code is still necessary for
1395          *  modules.
1396          */
1397
1398         pg = ftrace_pages = ftrace_pages_start;
1399
1400         cnt = num_to_init / ENTRIES_PER_PAGE;
1401         pr_info("ftrace: allocating %ld entries in %d pages\n",
1402                 num_to_init, cnt + 1);
1403
1404         for (i = 0; i < cnt; i++) {
1405                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1406
1407                 /* If we fail, we'll try later anyway */
1408                 if (!pg->next)
1409                         break;
1410
1411                 pg = pg->next;
1412         }
1413
1414         return 0;
1415 }
1416
1417 enum {
1418         FTRACE_ITER_FILTER      = (1 << 0),
1419         FTRACE_ITER_NOTRACE     = (1 << 1),
1420         FTRACE_ITER_PRINTALL    = (1 << 2),
1421         FTRACE_ITER_HASH        = (1 << 3),
1422 };
1423
1424 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1425
1426 struct ftrace_iterator {
1427         loff_t                          pos;
1428         loff_t                          func_pos;
1429         struct ftrace_page              *pg;
1430         struct dyn_ftrace               *func;
1431         struct ftrace_func_probe        *probe;
1432         struct trace_parser             parser;
1433         int                             hidx;
1434         int                             idx;
1435         unsigned                        flags;
1436 };
1437
1438 static void *
1439 t_hash_next(struct seq_file *m, loff_t *pos)
1440 {
1441         struct ftrace_iterator *iter = m->private;
1442         struct hlist_node *hnd = NULL;
1443         struct hlist_head *hhd;
1444
1445         (*pos)++;
1446         iter->pos = *pos;
1447
1448         if (iter->probe)
1449                 hnd = &iter->probe->node;
1450  retry:
1451         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1452                 return NULL;
1453
1454         hhd = &ftrace_func_hash[iter->hidx];
1455
1456         if (hlist_empty(hhd)) {
1457                 iter->hidx++;
1458                 hnd = NULL;
1459                 goto retry;
1460         }
1461
1462         if (!hnd)
1463                 hnd = hhd->first;
1464         else {
1465                 hnd = hnd->next;
1466                 if (!hnd) {
1467                         iter->hidx++;
1468                         goto retry;
1469                 }
1470         }
1471
1472         if (WARN_ON_ONCE(!hnd))
1473                 return NULL;
1474
1475         iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
1476
1477         return iter;
1478 }
1479
1480 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1481 {
1482         struct ftrace_iterator *iter = m->private;
1483         void *p = NULL;
1484         loff_t l;
1485
1486         if (iter->func_pos > *pos)
1487                 return NULL;
1488
1489         iter->hidx = 0;
1490         for (l = 0; l <= (*pos - iter->func_pos); ) {
1491                 p = t_hash_next(m, &l);
1492                 if (!p)
1493                         break;
1494         }
1495         if (!p)
1496                 return NULL;
1497
1498         /* Only set this if we have an item */
1499         iter->flags |= FTRACE_ITER_HASH;
1500
1501         return iter;
1502 }
1503
1504 static int
1505 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
1506 {
1507         struct ftrace_func_probe *rec;
1508
1509         rec = iter->probe;
1510         if (WARN_ON_ONCE(!rec))
1511                 return -EIO;
1512
1513         if (rec->ops->print)
1514                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1515
1516         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1517
1518         if (rec->data)
1519                 seq_printf(m, ":%p", rec->data);
1520         seq_putc(m, '\n');
1521
1522         return 0;
1523 }
1524
1525 static void *
1526 t_next(struct seq_file *m, void *v, loff_t *pos)
1527 {
1528         struct ftrace_iterator *iter = m->private;
1529         struct dyn_ftrace *rec = NULL;
1530
1531         if (unlikely(ftrace_disabled))
1532                 return NULL;
1533
1534         if (iter->flags & FTRACE_ITER_HASH)
1535                 return t_hash_next(m, pos);
1536
1537         (*pos)++;
1538         iter->pos = iter->func_pos = *pos;
1539
1540         if (iter->flags & FTRACE_ITER_PRINTALL)
1541                 return t_hash_start(m, pos);
1542
1543  retry:
1544         if (iter->idx >= iter->pg->index) {
1545                 if (iter->pg->next) {
1546                         iter->pg = iter->pg->next;
1547                         iter->idx = 0;
1548                         goto retry;
1549                 }
1550         } else {
1551                 rec = &iter->pg->records[iter->idx++];
1552                 if ((rec->flags & FTRACE_FL_FREE) ||
1553
1554                     ((iter->flags & FTRACE_ITER_FILTER) &&
1555                      !(rec->flags & FTRACE_FL_FILTER)) ||
1556
1557                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1558                      !ftrace_lookup_ip(&notrace_hash, rec->ip))) {
1559                         rec = NULL;
1560                         goto retry;
1561                 }
1562         }
1563
1564         if (!rec)
1565                 return t_hash_start(m, pos);
1566
1567         iter->func = rec;
1568
1569         return iter;
1570 }
1571
1572 static void reset_iter_read(struct ftrace_iterator *iter)
1573 {
1574         iter->pos = 0;
1575         iter->func_pos = 0;
1576         iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
1577 }
1578
1579 static void *t_start(struct seq_file *m, loff_t *pos)
1580 {
1581         struct ftrace_iterator *iter = m->private;
1582         void *p = NULL;
1583         loff_t l;
1584
1585         mutex_lock(&ftrace_lock);
1586
1587         if (unlikely(ftrace_disabled))
1588                 return NULL;
1589
1590         /*
1591          * If an lseek was done, then reset and start from beginning.
1592          */
1593         if (*pos < iter->pos)
1594                 reset_iter_read(iter);
1595
1596         /*
1597          * For set_ftrace_filter reading, if we have the filter
1598          * off, we can short cut and just print out that all
1599          * functions are enabled.
1600          */
1601         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1602                 if (*pos > 0)
1603                         return t_hash_start(m, pos);
1604                 iter->flags |= FTRACE_ITER_PRINTALL;
1605                 /* reset in case of seek/pread */
1606                 iter->flags &= ~FTRACE_ITER_HASH;
1607                 return iter;
1608         }
1609
1610         if (iter->flags & FTRACE_ITER_HASH)
1611                 return t_hash_start(m, pos);
1612
1613         /*
1614          * Unfortunately, we need to restart at ftrace_pages_start
1615          * every time we let go of the ftrace_mutex. This is because
1616          * those pointers can change without the lock.
1617          */
1618         iter->pg = ftrace_pages_start;
1619         iter->idx = 0;
1620         for (l = 0; l <= *pos; ) {
1621                 p = t_next(m, p, &l);
1622                 if (!p)
1623                         break;
1624         }
1625
1626         if (!p) {
1627                 if (iter->flags & FTRACE_ITER_FILTER)
1628                         return t_hash_start(m, pos);
1629
1630                 return NULL;
1631         }
1632
1633         return iter;
1634 }
1635
1636 static void t_stop(struct seq_file *m, void *p)
1637 {
1638         mutex_unlock(&ftrace_lock);
1639 }
1640
1641 static int t_show(struct seq_file *m, void *v)
1642 {
1643         struct ftrace_iterator *iter = m->private;
1644         struct dyn_ftrace *rec;
1645
1646         if (iter->flags & FTRACE_ITER_HASH)
1647                 return t_hash_show(m, iter);
1648
1649         if (iter->flags & FTRACE_ITER_PRINTALL) {
1650                 seq_printf(m, "#### all functions enabled ####\n");
1651                 return 0;
1652         }
1653
1654         rec = iter->func;
1655
1656         if (!rec)
1657                 return 0;
1658
1659         seq_printf(m, "%ps\n", (void *)rec->ip);
1660
1661         return 0;
1662 }
1663
1664 static const struct seq_operations show_ftrace_seq_ops = {
1665         .start = t_start,
1666         .next = t_next,
1667         .stop = t_stop,
1668         .show = t_show,
1669 };
1670
1671 static int
1672 ftrace_avail_open(struct inode *inode, struct file *file)
1673 {
1674         struct ftrace_iterator *iter;
1675         int ret;
1676
1677         if (unlikely(ftrace_disabled))
1678                 return -ENODEV;
1679
1680         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1681         if (!iter)
1682                 return -ENOMEM;
1683
1684         iter->pg = ftrace_pages_start;
1685
1686         ret = seq_open(file, &show_ftrace_seq_ops);
1687         if (!ret) {
1688                 struct seq_file *m = file->private_data;
1689
1690                 m->private = iter;
1691         } else {
1692                 kfree(iter);
1693         }
1694
1695         return ret;
1696 }
1697
1698 static void ftrace_filter_reset(int enable)
1699 {
1700         struct ftrace_page *pg;
1701         struct dyn_ftrace *rec;
1702
1703         mutex_lock(&ftrace_lock);
1704         if (enable) {
1705                 ftrace_filtered = 0;
1706                 do_for_each_ftrace_rec(pg, rec) {
1707                         rec->flags &= ~FTRACE_FL_FILTER;
1708                 } while_for_each_ftrace_rec();
1709         } else
1710                 ftrace_hash_clear(&notrace_hash);
1711         mutex_unlock(&ftrace_lock);
1712 }
1713
1714 static int
1715 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1716 {
1717         struct ftrace_iterator *iter;
1718         int ret = 0;
1719
1720         if (unlikely(ftrace_disabled))
1721                 return -ENODEV;
1722
1723         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1724         if (!iter)
1725                 return -ENOMEM;
1726
1727         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1728                 kfree(iter);
1729                 return -ENOMEM;
1730         }
1731
1732         mutex_lock(&ftrace_regex_lock);
1733         if ((file->f_mode & FMODE_WRITE) &&
1734             (file->f_flags & O_TRUNC))
1735                 ftrace_filter_reset(enable);
1736
1737         if (file->f_mode & FMODE_READ) {
1738                 iter->pg = ftrace_pages_start;
1739                 iter->flags = enable ? FTRACE_ITER_FILTER :
1740                         FTRACE_ITER_NOTRACE;
1741
1742                 ret = seq_open(file, &show_ftrace_seq_ops);
1743                 if (!ret) {
1744                         struct seq_file *m = file->private_data;
1745                         m->private = iter;
1746                 } else {
1747                         trace_parser_put(&iter->parser);
1748                         kfree(iter);
1749                 }
1750         } else
1751                 file->private_data = iter;
1752         mutex_unlock(&ftrace_regex_lock);
1753
1754         return ret;
1755 }
1756
1757 static int
1758 ftrace_filter_open(struct inode *inode, struct file *file)
1759 {
1760         return ftrace_regex_open(inode, file, 1);
1761 }
1762
1763 static int
1764 ftrace_notrace_open(struct inode *inode, struct file *file)
1765 {
1766         return ftrace_regex_open(inode, file, 0);
1767 }
1768
1769 static loff_t
1770 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1771 {
1772         loff_t ret;
1773
1774         if (file->f_mode & FMODE_READ)
1775                 ret = seq_lseek(file, offset, origin);
1776         else
1777                 file->f_pos = ret = 1;
1778
1779         return ret;
1780 }
1781
1782 static int ftrace_match(char *str, char *regex, int len, int type)
1783 {
1784         int matched = 0;
1785         int slen;
1786
1787         switch (type) {
1788         case MATCH_FULL:
1789                 if (strcmp(str, regex) == 0)
1790                         matched = 1;
1791                 break;
1792         case MATCH_FRONT_ONLY:
1793                 if (strncmp(str, regex, len) == 0)
1794                         matched = 1;
1795                 break;
1796         case MATCH_MIDDLE_ONLY:
1797                 if (strstr(str, regex))
1798                         matched = 1;
1799                 break;
1800         case MATCH_END_ONLY:
1801                 slen = strlen(str);
1802                 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
1803                         matched = 1;
1804                 break;
1805         }
1806
1807         return matched;
1808 }
1809
1810 static int
1811 update_record(struct dyn_ftrace *rec, int enable, int not)
1812 {
1813         struct ftrace_func_entry *entry;
1814         struct ftrace_hash *hash = &notrace_hash;
1815         int ret = 0;
1816
1817         if (enable) {
1818                 if (not)
1819                         rec->flags &= ~FTRACE_FL_FILTER;
1820                 else
1821                         rec->flags |= FTRACE_FL_FILTER;
1822         } else {
1823                 if (not) {
1824                         /* Do nothing if it doesn't exist */
1825                         entry = ftrace_lookup_ip(hash, rec->ip);
1826                         if (!entry)
1827                                 return 0;
1828
1829                         remove_hash_entry(hash, entry);
1830                 } else {
1831                         /* Do nothing if it exists */
1832                         entry = ftrace_lookup_ip(hash, rec->ip);
1833                         if (entry)
1834                                 return 0;
1835
1836                         ret = add_hash_entry(hash, rec->ip);
1837                 }
1838         }
1839         return ret;
1840 }
1841
1842 static int
1843 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
1844                     char *regex, int len, int type)
1845 {
1846         char str[KSYM_SYMBOL_LEN];
1847         char *modname;
1848
1849         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1850
1851         if (mod) {
1852                 /* module lookup requires matching the module */
1853                 if (!modname || strcmp(modname, mod))
1854                         return 0;
1855
1856                 /* blank search means to match all funcs in the mod */
1857                 if (!len)
1858                         return 1;
1859         }
1860
1861         return ftrace_match(str, regex, len, type);
1862 }
1863
1864 static int match_records(char *buff, int len, char *mod, int enable, int not)
1865 {
1866         unsigned search_len = 0;
1867         struct ftrace_page *pg;
1868         struct dyn_ftrace *rec;
1869         int type = MATCH_FULL;
1870         char *search = buff;
1871         int found = 0;
1872         int ret;
1873
1874         if (len) {
1875                 type = filter_parse_regex(buff, len, &search, &not);
1876                 search_len = strlen(search);
1877         }
1878
1879         mutex_lock(&ftrace_lock);
1880
1881         if (unlikely(ftrace_disabled))
1882                 goto out_unlock;
1883
1884         do_for_each_ftrace_rec(pg, rec) {
1885
1886                 if (ftrace_match_record(rec, mod, search, search_len, type)) {
1887                         ret = update_record(rec, enable, not);
1888                         if (ret < 0) {
1889                                 found = ret;
1890                                 goto out_unlock;
1891                         }
1892                         found = 1;
1893                 }
1894                 /*
1895                  * Only enable filtering if we have a function that
1896                  * is filtered on.
1897                  */
1898                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1899                         ftrace_filtered = 1;
1900
1901         } while_for_each_ftrace_rec();
1902  out_unlock:
1903         mutex_unlock(&ftrace_lock);
1904
1905         return found;
1906 }
1907
1908 static int
1909 ftrace_match_records(char *buff, int len, int enable)
1910 {
1911         return match_records(buff, len, NULL, enable, 0);
1912 }
1913
1914 static int ftrace_match_module_records(char *buff, char *mod, int enable)
1915 {
1916         int not = 0;
1917
1918         /* blank or '*' mean the same */
1919         if (strcmp(buff, "*") == 0)
1920                 buff[0] = 0;
1921
1922         /* handle the case of 'dont filter this module' */
1923         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1924                 buff[0] = 0;
1925                 not = 1;
1926         }
1927
1928         return match_records(buff, strlen(buff), mod, enable, not);
1929 }
1930
1931 /*
1932  * We register the module command as a template to show others how
1933  * to register the a command as well.
1934  */
1935
1936 static int
1937 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1938 {
1939         char *mod;
1940         int ret = -EINVAL;
1941
1942         /*
1943          * cmd == 'mod' because we only registered this func
1944          * for the 'mod' ftrace_func_command.
1945          * But if you register one func with multiple commands,
1946          * you can tell which command was used by the cmd
1947          * parameter.
1948          */
1949
1950         /* we must have a module name */
1951         if (!param)
1952                 return ret;
1953
1954         mod = strsep(&param, ":");
1955         if (!strlen(mod))
1956                 return ret;
1957
1958         ret = ftrace_match_module_records(func, mod, enable);
1959         if (!ret)
1960                 ret = -EINVAL;
1961         if (ret < 0)
1962                 return ret;
1963
1964         return 0;
1965 }
1966
1967 static struct ftrace_func_command ftrace_mod_cmd = {
1968         .name                   = "mod",
1969         .func                   = ftrace_mod_callback,
1970 };
1971
1972 static int __init ftrace_mod_cmd_init(void)
1973 {
1974         return register_ftrace_command(&ftrace_mod_cmd);
1975 }
1976 device_initcall(ftrace_mod_cmd_init);
1977
1978 static void
1979 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1980 {
1981         struct ftrace_func_probe *entry;
1982         struct hlist_head *hhd;
1983         struct hlist_node *n;
1984         unsigned long key;
1985
1986         key = hash_long(ip, FTRACE_HASH_BITS);
1987
1988         hhd = &ftrace_func_hash[key];
1989
1990         if (hlist_empty(hhd))
1991                 return;
1992
1993         /*
1994          * Disable preemption for these calls to prevent a RCU grace
1995          * period. This syncs the hash iteration and freeing of items
1996          * on the hash. rcu_read_lock is too dangerous here.
1997          */
1998         preempt_disable_notrace();
1999         hlist_for_each_entry_rcu(entry, n, hhd, node) {
2000                 if (entry->ip == ip)
2001                         entry->ops->func(ip, parent_ip, &entry->data);
2002         }
2003         preempt_enable_notrace();
2004 }
2005
2006 static struct ftrace_ops trace_probe_ops __read_mostly =
2007 {
2008         .func           = function_trace_probe_call,
2009 };
2010
2011 static int ftrace_probe_registered;
2012
2013 static void __enable_ftrace_function_probe(void)
2014 {
2015         int i;
2016
2017         if (ftrace_probe_registered)
2018                 return;
2019
2020         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2021                 struct hlist_head *hhd = &ftrace_func_hash[i];
2022                 if (hhd->first)
2023                         break;
2024         }
2025         /* Nothing registered? */
2026         if (i == FTRACE_FUNC_HASHSIZE)
2027                 return;
2028
2029         __register_ftrace_function(&trace_probe_ops);
2030         ftrace_startup(0);
2031         ftrace_probe_registered = 1;
2032 }
2033
2034 static void __disable_ftrace_function_probe(void)
2035 {
2036         int i;
2037
2038         if (!ftrace_probe_registered)
2039                 return;
2040
2041         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2042                 struct hlist_head *hhd = &ftrace_func_hash[i];
2043                 if (hhd->first)
2044                         return;
2045         }
2046
2047         /* no more funcs left */
2048         __unregister_ftrace_function(&trace_probe_ops);
2049         ftrace_shutdown(0);
2050         ftrace_probe_registered = 0;
2051 }
2052
2053
2054 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2055 {
2056         struct ftrace_func_probe *entry =
2057                 container_of(rhp, struct ftrace_func_probe, rcu);
2058
2059         if (entry->ops->free)
2060                 entry->ops->free(&entry->data);
2061         kfree(entry);
2062 }
2063
2064
2065 int
2066 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2067                               void *data)
2068 {
2069         struct ftrace_func_probe *entry;
2070         struct ftrace_page *pg;
2071         struct dyn_ftrace *rec;
2072         int type, len, not;
2073         unsigned long key;
2074         int count = 0;
2075         char *search;
2076
2077         type = filter_parse_regex(glob, strlen(glob), &search, &not);
2078         len = strlen(search);
2079
2080         /* we do not support '!' for function probes */
2081         if (WARN_ON(not))
2082                 return -EINVAL;
2083
2084         mutex_lock(&ftrace_lock);
2085
2086         if (unlikely(ftrace_disabled))
2087                 goto out_unlock;
2088
2089         do_for_each_ftrace_rec(pg, rec) {
2090
2091                 if (!ftrace_match_record(rec, NULL, search, len, type))
2092                         continue;
2093
2094                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2095                 if (!entry) {
2096                         /* If we did not process any, then return error */
2097                         if (!count)
2098                                 count = -ENOMEM;
2099                         goto out_unlock;
2100                 }
2101
2102                 count++;
2103
2104                 entry->data = data;
2105
2106                 /*
2107                  * The caller might want to do something special
2108                  * for each function we find. We call the callback
2109                  * to give the caller an opportunity to do so.
2110                  */
2111                 if (ops->callback) {
2112                         if (ops->callback(rec->ip, &entry->data) < 0) {
2113                                 /* caller does not like this func */
2114                                 kfree(entry);
2115                                 continue;
2116                         }
2117                 }
2118
2119                 entry->ops = ops;
2120                 entry->ip = rec->ip;
2121
2122                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2123                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2124
2125         } while_for_each_ftrace_rec();
2126         __enable_ftrace_function_probe();
2127
2128  out_unlock:
2129         mutex_unlock(&ftrace_lock);
2130
2131         return count;
2132 }
2133
2134 enum {
2135         PROBE_TEST_FUNC         = 1,
2136         PROBE_TEST_DATA         = 2
2137 };
2138
2139 static void
2140 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2141                                   void *data, int flags)
2142 {
2143         struct ftrace_func_probe *entry;
2144         struct hlist_node *n, *tmp;
2145         char str[KSYM_SYMBOL_LEN];
2146         int type = MATCH_FULL;
2147         int i, len = 0;
2148         char *search;
2149
2150         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2151                 glob = NULL;
2152         else if (glob) {
2153                 int not;
2154
2155                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2156                 len = strlen(search);
2157
2158                 /* we do not support '!' for function probes */
2159                 if (WARN_ON(not))
2160                         return;
2161         }
2162
2163         mutex_lock(&ftrace_lock);
2164         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2165                 struct hlist_head *hhd = &ftrace_func_hash[i];
2166
2167                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2168
2169                         /* break up if statements for readability */
2170                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2171                                 continue;
2172
2173                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2174                                 continue;
2175
2176                         /* do this last, since it is the most expensive */
2177                         if (glob) {
2178                                 kallsyms_lookup(entry->ip, NULL, NULL,
2179                                                 NULL, str);
2180                                 if (!ftrace_match(str, glob, len, type))
2181                                         continue;
2182                         }
2183
2184                         hlist_del(&entry->node);
2185                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2186                 }
2187         }
2188         __disable_ftrace_function_probe();
2189         mutex_unlock(&ftrace_lock);
2190 }
2191
2192 void
2193 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2194                                 void *data)
2195 {
2196         __unregister_ftrace_function_probe(glob, ops, data,
2197                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2198 }
2199
2200 void
2201 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2202 {
2203         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2204 }
2205
2206 void unregister_ftrace_function_probe_all(char *glob)
2207 {
2208         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2209 }
2210
2211 static LIST_HEAD(ftrace_commands);
2212 static DEFINE_MUTEX(ftrace_cmd_mutex);
2213
2214 int register_ftrace_command(struct ftrace_func_command *cmd)
2215 {
2216         struct ftrace_func_command *p;
2217         int ret = 0;
2218
2219         mutex_lock(&ftrace_cmd_mutex);
2220         list_for_each_entry(p, &ftrace_commands, list) {
2221                 if (strcmp(cmd->name, p->name) == 0) {
2222                         ret = -EBUSY;
2223                         goto out_unlock;
2224                 }
2225         }
2226         list_add(&cmd->list, &ftrace_commands);
2227  out_unlock:
2228         mutex_unlock(&ftrace_cmd_mutex);
2229
2230         return ret;
2231 }
2232
2233 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2234 {
2235         struct ftrace_func_command *p, *n;
2236         int ret = -ENODEV;
2237
2238         mutex_lock(&ftrace_cmd_mutex);
2239         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2240                 if (strcmp(cmd->name, p->name) == 0) {
2241                         ret = 0;
2242                         list_del_init(&p->list);
2243                         goto out_unlock;
2244                 }
2245         }
2246  out_unlock:
2247         mutex_unlock(&ftrace_cmd_mutex);
2248
2249         return ret;
2250 }
2251
2252 static int ftrace_process_regex(char *buff, int len, int enable)
2253 {
2254         char *func, *command, *next = buff;
2255         struct ftrace_func_command *p;
2256         int ret;
2257
2258         func = strsep(&next, ":");
2259
2260         if (!next) {
2261                 ret = ftrace_match_records(func, len, enable);
2262                 if (!ret)
2263                         ret = -EINVAL;
2264                 if (ret < 0)
2265                         return ret;
2266                 return 0;
2267         }
2268
2269         /* command found */
2270
2271         command = strsep(&next, ":");
2272
2273         mutex_lock(&ftrace_cmd_mutex);
2274         list_for_each_entry(p, &ftrace_commands, list) {
2275                 if (strcmp(p->name, command) == 0) {
2276                         ret = p->func(func, command, next, enable);
2277                         goto out_unlock;
2278                 }
2279         }
2280  out_unlock:
2281         mutex_unlock(&ftrace_cmd_mutex);
2282
2283         return ret;
2284 }
2285
2286 static ssize_t
2287 ftrace_regex_write(struct file *file, const char __user *ubuf,
2288                    size_t cnt, loff_t *ppos, int enable)
2289 {
2290         struct ftrace_iterator *iter;
2291         struct trace_parser *parser;
2292         ssize_t ret, read;
2293
2294         if (!cnt)
2295                 return 0;
2296
2297         mutex_lock(&ftrace_regex_lock);
2298
2299         ret = -ENODEV;
2300         if (unlikely(ftrace_disabled))
2301                 goto out_unlock;
2302
2303         if (file->f_mode & FMODE_READ) {
2304                 struct seq_file *m = file->private_data;
2305                 iter = m->private;
2306         } else
2307                 iter = file->private_data;
2308
2309         parser = &iter->parser;
2310         read = trace_get_user(parser, ubuf, cnt, ppos);
2311
2312         if (read >= 0 && trace_parser_loaded(parser) &&
2313             !trace_parser_cont(parser)) {
2314                 ret = ftrace_process_regex(parser->buffer,
2315                                            parser->idx, enable);
2316                 trace_parser_clear(parser);
2317                 if (ret)
2318                         goto out_unlock;
2319         }
2320
2321         ret = read;
2322 out_unlock:
2323         mutex_unlock(&ftrace_regex_lock);
2324
2325         return ret;
2326 }
2327
2328 static ssize_t
2329 ftrace_filter_write(struct file *file, const char __user *ubuf,
2330                     size_t cnt, loff_t *ppos)
2331 {
2332         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2333 }
2334
2335 static ssize_t
2336 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2337                      size_t cnt, loff_t *ppos)
2338 {
2339         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2340 }
2341
2342 static void
2343 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2344 {
2345         if (unlikely(ftrace_disabled))
2346                 return;
2347
2348         mutex_lock(&ftrace_regex_lock);
2349         if (reset)
2350                 ftrace_filter_reset(enable);
2351         if (buf)
2352                 ftrace_match_records(buf, len, enable);
2353         mutex_unlock(&ftrace_regex_lock);
2354 }
2355
2356 /**
2357  * ftrace_set_filter - set a function to filter on in ftrace
2358  * @buf - the string that holds the function filter text.
2359  * @len - the length of the string.
2360  * @reset - non zero to reset all filters before applying this filter.
2361  *
2362  * Filters denote which functions should be enabled when tracing is enabled.
2363  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2364  */
2365 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2366 {
2367         ftrace_set_regex(buf, len, reset, 1);
2368 }
2369
2370 /**
2371  * ftrace_set_notrace - set a function to not trace in ftrace
2372  * @buf - the string that holds the function notrace text.
2373  * @len - the length of the string.
2374  * @reset - non zero to reset all filters before applying this filter.
2375  *
2376  * Notrace Filters denote which functions should not be enabled when tracing
2377  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2378  * for tracing.
2379  */
2380 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2381 {
2382         ftrace_set_regex(buf, len, reset, 0);
2383 }
2384
2385 /*
2386  * command line interface to allow users to set filters on boot up.
2387  */
2388 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
2389 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2390 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2391
2392 static int __init set_ftrace_notrace(char *str)
2393 {
2394         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2395         return 1;
2396 }
2397 __setup("ftrace_notrace=", set_ftrace_notrace);
2398
2399 static int __init set_ftrace_filter(char *str)
2400 {
2401         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2402         return 1;
2403 }
2404 __setup("ftrace_filter=", set_ftrace_filter);
2405
2406 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2407 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2408 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
2409
2410 static int __init set_graph_function(char *str)
2411 {
2412         strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2413         return 1;
2414 }
2415 __setup("ftrace_graph_filter=", set_graph_function);
2416
2417 static void __init set_ftrace_early_graph(char *buf)
2418 {
2419         int ret;
2420         char *func;
2421
2422         while (buf) {
2423                 func = strsep(&buf, ",");
2424                 /* we allow only one expression at a time */
2425                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2426                                       func);
2427                 if (ret)
2428                         printk(KERN_DEBUG "ftrace: function %s not "
2429                                           "traceable\n", func);
2430         }
2431 }
2432 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2433
2434 static void __init set_ftrace_early_filter(char *buf, int enable)
2435 {
2436         char *func;
2437
2438         while (buf) {
2439                 func = strsep(&buf, ",");
2440                 ftrace_set_regex(func, strlen(func), 0, enable);
2441         }
2442 }
2443
2444 static void __init set_ftrace_early_filters(void)
2445 {
2446         if (ftrace_filter_buf[0])
2447                 set_ftrace_early_filter(ftrace_filter_buf, 1);
2448         if (ftrace_notrace_buf[0])
2449                 set_ftrace_early_filter(ftrace_notrace_buf, 0);
2450 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2451         if (ftrace_graph_buf[0])
2452                 set_ftrace_early_graph(ftrace_graph_buf);
2453 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2454 }
2455
2456 static int
2457 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2458 {
2459         struct seq_file *m = (struct seq_file *)file->private_data;
2460         struct ftrace_iterator *iter;
2461         struct trace_parser *parser;
2462
2463         mutex_lock(&ftrace_regex_lock);
2464         if (file->f_mode & FMODE_READ) {
2465                 iter = m->private;
2466
2467                 seq_release(inode, file);
2468         } else
2469                 iter = file->private_data;
2470
2471         parser = &iter->parser;
2472         if (trace_parser_loaded(parser)) {
2473                 parser->buffer[parser->idx] = 0;
2474                 ftrace_match_records(parser->buffer, parser->idx, enable);
2475         }
2476
2477         trace_parser_put(parser);
2478         kfree(iter);
2479
2480         if (file->f_mode & FMODE_WRITE) {
2481                 mutex_lock(&ftrace_lock);
2482                 if (ftrace_start_up && ftrace_enabled)
2483                         ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2484                 mutex_unlock(&ftrace_lock);
2485         }
2486
2487         mutex_unlock(&ftrace_regex_lock);
2488         return 0;
2489 }
2490
2491 static int
2492 ftrace_filter_release(struct inode *inode, struct file *file)
2493 {
2494         return ftrace_regex_release(inode, file, 1);
2495 }
2496
2497 static int
2498 ftrace_notrace_release(struct inode *inode, struct file *file)
2499 {
2500         return ftrace_regex_release(inode, file, 0);
2501 }
2502
2503 static const struct file_operations ftrace_avail_fops = {
2504         .open = ftrace_avail_open,
2505         .read = seq_read,
2506         .llseek = seq_lseek,
2507         .release = seq_release_private,
2508 };
2509
2510 static const struct file_operations ftrace_filter_fops = {
2511         .open = ftrace_filter_open,
2512         .read = seq_read,
2513         .write = ftrace_filter_write,
2514         .llseek = ftrace_regex_lseek,
2515         .release = ftrace_filter_release,
2516 };
2517
2518 static const struct file_operations ftrace_notrace_fops = {
2519         .open = ftrace_notrace_open,
2520         .read = seq_read,
2521         .write = ftrace_notrace_write,
2522         .llseek = ftrace_regex_lseek,
2523         .release = ftrace_notrace_release,
2524 };
2525
2526 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2527
2528 static DEFINE_MUTEX(graph_lock);
2529
2530 int ftrace_graph_count;
2531 int ftrace_graph_filter_enabled;
2532 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2533
2534 static void *
2535 __g_next(struct seq_file *m, loff_t *pos)
2536 {
2537         if (*pos >= ftrace_graph_count)
2538                 return NULL;
2539         return &ftrace_graph_funcs[*pos];
2540 }
2541
2542 static void *
2543 g_next(struct seq_file *m, void *v, loff_t *pos)
2544 {
2545         (*pos)++;
2546         return __g_next(m, pos);
2547 }
2548
2549 static void *g_start(struct seq_file *m, loff_t *pos)
2550 {
2551         mutex_lock(&graph_lock);
2552
2553         /* Nothing, tell g_show to print all functions are enabled */
2554         if (!ftrace_graph_filter_enabled && !*pos)
2555                 return (void *)1;
2556
2557         return __g_next(m, pos);
2558 }
2559
2560 static void g_stop(struct seq_file *m, void *p)
2561 {
2562         mutex_unlock(&graph_lock);
2563 }
2564
2565 static int g_show(struct seq_file *m, void *v)
2566 {
2567         unsigned long *ptr = v;
2568
2569         if (!ptr)
2570                 return 0;
2571
2572         if (ptr == (unsigned long *)1) {
2573                 seq_printf(m, "#### all functions enabled ####\n");
2574                 return 0;
2575         }
2576
2577         seq_printf(m, "%ps\n", (void *)*ptr);
2578
2579         return 0;
2580 }
2581
2582 static const struct seq_operations ftrace_graph_seq_ops = {
2583         .start = g_start,
2584         .next = g_next,
2585         .stop = g_stop,
2586         .show = g_show,
2587 };
2588
2589 static int
2590 ftrace_graph_open(struct inode *inode, struct file *file)
2591 {
2592         int ret = 0;
2593
2594         if (unlikely(ftrace_disabled))
2595                 return -ENODEV;
2596
2597         mutex_lock(&graph_lock);
2598         if ((file->f_mode & FMODE_WRITE) &&
2599             (file->f_flags & O_TRUNC)) {
2600                 ftrace_graph_filter_enabled = 0;
2601                 ftrace_graph_count = 0;
2602                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2603         }
2604         mutex_unlock(&graph_lock);
2605
2606         if (file->f_mode & FMODE_READ)
2607                 ret = seq_open(file, &ftrace_graph_seq_ops);
2608
2609         return ret;
2610 }
2611
2612 static int
2613 ftrace_graph_release(struct inode *inode, struct file *file)
2614 {
2615         if (file->f_mode & FMODE_READ)
2616                 seq_release(inode, file);
2617         return 0;
2618 }
2619
2620 static int
2621 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2622 {
2623         struct dyn_ftrace *rec;
2624         struct ftrace_page *pg;
2625         int search_len;
2626         int fail = 1;
2627         int type, not;
2628         char *search;
2629         bool exists;
2630         int i;
2631
2632         /* decode regex */
2633         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2634         if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
2635                 return -EBUSY;
2636
2637         search_len = strlen(search);
2638
2639         mutex_lock(&ftrace_lock);
2640
2641         if (unlikely(ftrace_disabled)) {
2642                 mutex_unlock(&ftrace_lock);
2643                 return -ENODEV;
2644         }
2645
2646         do_for_each_ftrace_rec(pg, rec) {
2647
2648                 if (rec->flags & FTRACE_FL_FREE)
2649                         continue;
2650
2651                 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
2652                         /* if it is in the array */
2653                         exists = false;
2654                         for (i = 0; i < *idx; i++) {
2655                                 if (array[i] == rec->ip) {
2656                                         exists = true;
2657                                         break;
2658                                 }
2659                         }
2660
2661                         if (!not) {
2662                                 fail = 0;
2663                                 if (!exists) {
2664                                         array[(*idx)++] = rec->ip;
2665                                         if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2666                                                 goto out;
2667                                 }
2668                         } else {
2669                                 if (exists) {
2670                                         array[i] = array[--(*idx)];
2671                                         array[*idx] = 0;
2672                                         fail = 0;
2673                                 }
2674                         }
2675                 }
2676         } while_for_each_ftrace_rec();
2677 out:
2678         mutex_unlock(&ftrace_lock);
2679
2680         if (fail)
2681                 return -EINVAL;
2682
2683         ftrace_graph_filter_enabled = 1;
2684         return 0;
2685 }
2686
2687 static ssize_t
2688 ftrace_graph_write(struct file *file, const char __user *ubuf,
2689                    size_t cnt, loff_t *ppos)
2690 {
2691         struct trace_parser parser;
2692         ssize_t read, ret;
2693
2694         if (!cnt)
2695                 return 0;
2696
2697         mutex_lock(&graph_lock);
2698
2699         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2700                 ret = -ENOMEM;
2701                 goto out_unlock;
2702         }
2703
2704         read = trace_get_user(&parser, ubuf, cnt, ppos);
2705
2706         if (read >= 0 && trace_parser_loaded((&parser))) {
2707                 parser.buffer[parser.idx] = 0;
2708
2709                 /* we allow only one expression at a time */
2710                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2711                                         parser.buffer);
2712                 if (ret)
2713                         goto out_free;
2714         }
2715
2716         ret = read;
2717
2718 out_free:
2719         trace_parser_put(&parser);
2720 out_unlock:
2721         mutex_unlock(&graph_lock);
2722
2723         return ret;
2724 }
2725
2726 static const struct file_operations ftrace_graph_fops = {
2727         .open           = ftrace_graph_open,
2728         .read           = seq_read,
2729         .write          = ftrace_graph_write,
2730         .release        = ftrace_graph_release,
2731         .llseek         = seq_lseek,
2732 };
2733 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2734
2735 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2736 {
2737
2738         trace_create_file("available_filter_functions", 0444,
2739                         d_tracer, NULL, &ftrace_avail_fops);
2740
2741         trace_create_file("set_ftrace_filter", 0644, d_tracer,
2742                         NULL, &ftrace_filter_fops);
2743
2744         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2745                                     NULL, &ftrace_notrace_fops);
2746
2747 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2748         trace_create_file("set_graph_function", 0444, d_tracer,
2749                                     NULL,
2750                                     &ftrace_graph_fops);
2751 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2752
2753         return 0;
2754 }
2755
2756 static int ftrace_process_locs(struct module *mod,
2757                                unsigned long *start,
2758                                unsigned long *end)
2759 {
2760         unsigned long *p;
2761         unsigned long addr;
2762
2763         mutex_lock(&ftrace_lock);
2764         p = start;
2765         while (p < end) {
2766                 addr = ftrace_call_adjust(*p++);
2767                 /*
2768                  * Some architecture linkers will pad between
2769                  * the different mcount_loc sections of different
2770                  * object files to satisfy alignments.
2771                  * Skip any NULL pointers.
2772                  */
2773                 if (!addr)
2774                         continue;
2775                 ftrace_record_ip(addr);
2776         }
2777
2778         ftrace_update_code(mod);
2779         mutex_unlock(&ftrace_lock);
2780
2781         return 0;
2782 }
2783
2784 #ifdef CONFIG_MODULES
2785 void ftrace_release_mod(struct module *mod)
2786 {
2787         struct dyn_ftrace *rec;
2788         struct ftrace_page *pg;
2789
2790         mutex_lock(&ftrace_lock);
2791
2792         if (ftrace_disabled)
2793                 goto out_unlock;
2794
2795         do_for_each_ftrace_rec(pg, rec) {
2796                 if (within_module_core(rec->ip, mod)) {
2797                         /*
2798                          * rec->ip is changed in ftrace_free_rec()
2799                          * It should not between s and e if record was freed.
2800                          */
2801                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2802                         ftrace_free_rec(rec);
2803                 }
2804         } while_for_each_ftrace_rec();
2805  out_unlock:
2806         mutex_unlock(&ftrace_lock);
2807 }
2808
2809 static void ftrace_init_module(struct module *mod,
2810                                unsigned long *start, unsigned long *end)
2811 {
2812         if (ftrace_disabled || start == end)
2813                 return;
2814         ftrace_process_locs(mod, start, end);
2815 }
2816
2817 static int ftrace_module_notify(struct notifier_block *self,
2818                                 unsigned long val, void *data)
2819 {
2820         struct module *mod = data;
2821
2822         switch (val) {
2823         case MODULE_STATE_COMING:
2824                 ftrace_init_module(mod, mod->ftrace_callsites,
2825                                    mod->ftrace_callsites +
2826                                    mod->num_ftrace_callsites);
2827                 break;
2828         case MODULE_STATE_GOING:
2829                 ftrace_release_mod(mod);
2830                 break;
2831         }
2832
2833         return 0;
2834 }
2835 #else
2836 static int ftrace_module_notify(struct notifier_block *self,
2837                                 unsigned long val, void *data)
2838 {
2839         return 0;
2840 }
2841 #endif /* CONFIG_MODULES */
2842
2843 struct notifier_block ftrace_module_nb = {
2844         .notifier_call = ftrace_module_notify,
2845         .priority = 0,
2846 };
2847
2848 extern unsigned long __start_mcount_loc[];
2849 extern unsigned long __stop_mcount_loc[];
2850
2851 void __init ftrace_init(void)
2852 {
2853         unsigned long count, addr, flags;
2854         int ret;
2855
2856         /* Keep the ftrace pointer to the stub */
2857         addr = (unsigned long)ftrace_stub;
2858
2859         local_irq_save(flags);
2860         ftrace_dyn_arch_init(&addr);
2861         local_irq_restore(flags);
2862
2863         /* ftrace_dyn_arch_init places the return code in addr */
2864         if (addr)
2865                 goto failed;
2866
2867         count = __stop_mcount_loc - __start_mcount_loc;
2868
2869         ret = ftrace_dyn_table_alloc(count);
2870         if (ret)
2871                 goto failed;
2872
2873         last_ftrace_enabled = ftrace_enabled = 1;
2874
2875         ret = ftrace_process_locs(NULL,
2876                                   __start_mcount_loc,
2877                                   __stop_mcount_loc);
2878
2879         ret = register_module_notifier(&ftrace_module_nb);
2880         if (ret)
2881                 pr_warning("Failed to register trace ftrace module notifier\n");
2882
2883         set_ftrace_early_filters();
2884
2885         return;
2886  failed:
2887         ftrace_disabled = 1;
2888 }
2889
2890 #else
2891
2892 static int __init ftrace_nodyn_init(void)
2893 {
2894         ftrace_enabled = 1;
2895         return 0;
2896 }
2897 device_initcall(ftrace_nodyn_init);
2898
2899 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2900 static inline void ftrace_startup_enable(int command) { }
2901 /* Keep as macros so we do not need to define the commands */
2902 # define ftrace_startup(command)        do { } while (0)
2903 # define ftrace_shutdown(command)       do { } while (0)
2904 # define ftrace_startup_sysctl()        do { } while (0)
2905 # define ftrace_shutdown_sysctl()       do { } while (0)
2906 #endif /* CONFIG_DYNAMIC_FTRACE */
2907
2908 static void clear_ftrace_swapper(void)
2909 {
2910         struct task_struct *p;
2911         int cpu;
2912
2913         get_online_cpus();
2914         for_each_online_cpu(cpu) {
2915                 p = idle_task(cpu);
2916                 clear_tsk_trace_trace(p);
2917         }
2918         put_online_cpus();
2919 }
2920
2921 static void set_ftrace_swapper(void)
2922 {
2923         struct task_struct *p;
2924         int cpu;
2925
2926         get_online_cpus();
2927         for_each_online_cpu(cpu) {
2928                 p = idle_task(cpu);
2929                 set_tsk_trace_trace(p);
2930         }
2931         put_online_cpus();
2932 }
2933
2934 static void clear_ftrace_pid(struct pid *pid)
2935 {
2936         struct task_struct *p;
2937
2938         rcu_read_lock();
2939         do_each_pid_task(pid, PIDTYPE_PID, p) {
2940                 clear_tsk_trace_trace(p);
2941         } while_each_pid_task(pid, PIDTYPE_PID, p);
2942         rcu_read_unlock();
2943
2944         put_pid(pid);
2945 }
2946
2947 static void set_ftrace_pid(struct pid *pid)
2948 {
2949         struct task_struct *p;
2950
2951         rcu_read_lock();
2952         do_each_pid_task(pid, PIDTYPE_PID, p) {
2953                 set_tsk_trace_trace(p);
2954         } while_each_pid_task(pid, PIDTYPE_PID, p);
2955         rcu_read_unlock();
2956 }
2957
2958 static void clear_ftrace_pid_task(struct pid *pid)
2959 {
2960         if (pid == ftrace_swapper_pid)
2961                 clear_ftrace_swapper();
2962         else
2963                 clear_ftrace_pid(pid);
2964 }
2965
2966 static void set_ftrace_pid_task(struct pid *pid)
2967 {
2968         if (pid == ftrace_swapper_pid)
2969                 set_ftrace_swapper();
2970         else
2971                 set_ftrace_pid(pid);
2972 }
2973
2974 static int ftrace_pid_add(int p)
2975 {
2976         struct pid *pid;
2977         struct ftrace_pid *fpid;
2978         int ret = -EINVAL;
2979
2980         mutex_lock(&ftrace_lock);
2981
2982         if (!p)
2983                 pid = ftrace_swapper_pid;
2984         else
2985                 pid = find_get_pid(p);
2986
2987         if (!pid)
2988                 goto out;
2989
2990         ret = 0;
2991
2992         list_for_each_entry(fpid, &ftrace_pids, list)
2993                 if (fpid->pid == pid)
2994                         goto out_put;
2995
2996         ret = -ENOMEM;
2997
2998         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
2999         if (!fpid)
3000                 goto out_put;
3001
3002         list_add(&fpid->list, &ftrace_pids);
3003         fpid->pid = pid;
3004
3005         set_ftrace_pid_task(pid);
3006
3007         ftrace_update_pid_func();
3008         ftrace_startup_enable(0);
3009
3010         mutex_unlock(&ftrace_lock);
3011         return 0;
3012
3013 out_put:
3014         if (pid != ftrace_swapper_pid)
3015                 put_pid(pid);
3016
3017 out:
3018         mutex_unlock(&ftrace_lock);
3019         return ret;
3020 }
3021
3022 static void ftrace_pid_reset(void)
3023 {
3024         struct ftrace_pid *fpid, *safe;
3025
3026         mutex_lock(&ftrace_lock);
3027         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
3028                 struct pid *pid = fpid->pid;
3029
3030                 clear_ftrace_pid_task(pid);
3031
3032                 list_del(&fpid->list);
3033                 kfree(fpid);
3034         }
3035
3036         ftrace_update_pid_func();
3037         ftrace_startup_enable(0);
3038
3039         mutex_unlock(&ftrace_lock);
3040 }
3041
3042 static void *fpid_start(struct seq_file *m, loff_t *pos)
3043 {
3044         mutex_lock(&ftrace_lock);
3045
3046         if (list_empty(&ftrace_pids) && (!*pos))
3047                 return (void *) 1;
3048
3049         return seq_list_start(&ftrace_pids, *pos);
3050 }
3051
3052 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
3053 {
3054         if (v == (void *)1)
3055                 return NULL;
3056
3057         return seq_list_next(v, &ftrace_pids, pos);
3058 }
3059
3060 static void fpid_stop(struct seq_file *m, void *p)
3061 {
3062         mutex_unlock(&ftrace_lock);
3063 }
3064
3065 static int fpid_show(struct seq_file *m, void *v)
3066 {
3067         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
3068
3069         if (v == (void *)1) {
3070                 seq_printf(m, "no pid\n");
3071                 return 0;
3072         }
3073
3074         if (fpid->pid == ftrace_swapper_pid)
3075                 seq_printf(m, "swapper tasks\n");
3076         else
3077                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
3078
3079         return 0;
3080 }
3081
3082 static const struct seq_operations ftrace_pid_sops = {
3083         .start = fpid_start,
3084         .next = fpid_next,
3085         .stop = fpid_stop,
3086         .show = fpid_show,
3087 };
3088
3089 static int
3090 ftrace_pid_open(struct inode *inode, struct file *file)
3091 {
3092         int ret = 0;
3093
3094         if ((file->f_mode & FMODE_WRITE) &&
3095             (file->f_flags & O_TRUNC))
3096                 ftrace_pid_reset();
3097
3098         if (file->f_mode & FMODE_READ)
3099                 ret = seq_open(file, &ftrace_pid_sops);
3100
3101         return ret;
3102 }
3103
3104 static ssize_t
3105 ftrace_pid_write(struct file *filp, const char __user *ubuf,
3106                    size_t cnt, loff_t *ppos)
3107 {
3108         char buf[64], *tmp;
3109         long val;
3110         int ret;
3111
3112         if (cnt >= sizeof(buf))
3113                 return -EINVAL;
3114
3115         if (copy_from_user(&buf, ubuf, cnt))
3116                 return -EFAULT;
3117
3118         buf[cnt] = 0;
3119
3120         /*
3121          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3122          * to clean the filter quietly.
3123          */
3124         tmp = strstrip(buf);
3125         if (strlen(tmp) == 0)
3126                 return 1;
3127
3128         ret = strict_strtol(tmp, 10, &val);
3129         if (ret < 0)
3130                 return ret;
3131
3132         ret = ftrace_pid_add(val);
3133
3134         return ret ? ret : cnt;
3135 }
3136
3137 static int
3138 ftrace_pid_release(struct inode *inode, struct file *file)
3139 {
3140         if (file->f_mode & FMODE_READ)
3141                 seq_release(inode, file);
3142
3143         return 0;
3144 }
3145
3146 static const struct file_operations ftrace_pid_fops = {
3147         .open           = ftrace_pid_open,
3148         .write          = ftrace_pid_write,
3149         .read           = seq_read,
3150         .llseek         = seq_lseek,
3151         .release        = ftrace_pid_release,
3152 };
3153
3154 static __init int ftrace_init_debugfs(void)
3155 {
3156         struct dentry *d_tracer;
3157
3158         d_tracer = tracing_init_dentry();
3159         if (!d_tracer)
3160                 return 0;
3161
3162         ftrace_init_dyn_debugfs(d_tracer);
3163
3164         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3165                             NULL, &ftrace_pid_fops);
3166
3167         ftrace_profile_debugfs(d_tracer);
3168
3169         return 0;
3170 }
3171 fs_initcall(ftrace_init_debugfs);
3172
3173 /**
3174  * ftrace_kill - kill ftrace
3175  *
3176  * This function should be used by panic code. It stops ftrace
3177  * but in a not so nice way. If you need to simply kill ftrace
3178  * from a non-atomic section, use ftrace_kill.
3179  */
3180 void ftrace_kill(void)
3181 {
3182         ftrace_disabled = 1;
3183         ftrace_enabled = 0;
3184         clear_ftrace_function();
3185 }
3186
3187 /**
3188  * register_ftrace_function - register a function for profiling
3189  * @ops - ops structure that holds the function for profiling.
3190  *
3191  * Register a function to be called by all functions in the
3192  * kernel.
3193  *
3194  * Note: @ops->func and all the functions it calls must be labeled
3195  *       with "notrace", otherwise it will go into a
3196  *       recursive loop.
3197  */
3198 int register_ftrace_function(struct ftrace_ops *ops)
3199 {
3200         int ret = -1;
3201
3202         mutex_lock(&ftrace_lock);
3203
3204         if (unlikely(ftrace_disabled))
3205                 goto out_unlock;
3206
3207         ret = __register_ftrace_function(ops);
3208         ftrace_startup(0);
3209
3210  out_unlock:
3211         mutex_unlock(&ftrace_lock);
3212         return ret;
3213 }
3214
3215 /**
3216  * unregister_ftrace_function - unregister a function for profiling.
3217  * @ops - ops structure that holds the function to unregister
3218  *
3219  * Unregister a function that was added to be called by ftrace profiling.
3220  */
3221 int unregister_ftrace_function(struct ftrace_ops *ops)
3222 {
3223         int ret;
3224
3225         mutex_lock(&ftrace_lock);
3226         ret = __unregister_ftrace_function(ops);
3227         ftrace_shutdown(0);
3228         mutex_unlock(&ftrace_lock);
3229
3230         return ret;
3231 }
3232
3233 int
3234 ftrace_enable_sysctl(struct ctl_table *table, int write,
3235                      void __user *buffer, size_t *lenp,
3236                      loff_t *ppos)
3237 {
3238         int ret = -ENODEV;
3239
3240         mutex_lock(&ftrace_lock);
3241
3242         if (unlikely(ftrace_disabled))
3243                 goto out;
3244
3245         ret = proc_dointvec(table, write, buffer, lenp, ppos);
3246
3247         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3248                 goto out;
3249
3250         last_ftrace_enabled = !!ftrace_enabled;
3251
3252         if (ftrace_enabled) {
3253
3254                 ftrace_startup_sysctl();
3255
3256                 /* we are starting ftrace again */
3257                 if (ftrace_list != &ftrace_list_end) {
3258                         if (ftrace_list->next == &ftrace_list_end)
3259                                 ftrace_trace_function = ftrace_list->func;
3260                         else
3261                                 ftrace_trace_function = ftrace_list_func;
3262                 }
3263
3264         } else {
3265                 /* stopping ftrace calls (just send to ftrace_stub) */
3266                 ftrace_trace_function = ftrace_stub;
3267
3268                 ftrace_shutdown_sysctl();
3269         }
3270
3271  out:
3272         mutex_unlock(&ftrace_lock);
3273         return ret;
3274 }
3275
3276 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3277
3278 static int ftrace_graph_active;
3279 static struct notifier_block ftrace_suspend_notifier;
3280
3281 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3282 {
3283         return 0;
3284 }
3285
3286 /* The callbacks that hook a function */
3287 trace_func_graph_ret_t ftrace_graph_return =
3288                         (trace_func_graph_ret_t)ftrace_stub;
3289 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3290
3291 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3292 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3293 {
3294         int i;
3295         int ret = 0;
3296         unsigned long flags;
3297         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3298         struct task_struct *g, *t;
3299
3300         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3301                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3302                                         * sizeof(struct ftrace_ret_stack),
3303                                         GFP_KERNEL);
3304                 if (!ret_stack_list[i]) {
3305                         start = 0;
3306                         end = i;
3307                         ret = -ENOMEM;
3308                         goto free;
3309                 }
3310         }
3311
3312         read_lock_irqsave(&tasklist_lock, flags);
3313         do_each_thread(g, t) {
3314                 if (start == end) {
3315                         ret = -EAGAIN;
3316                         goto unlock;
3317                 }
3318
3319                 if (t->ret_stack == NULL) {
3320                         atomic_set(&t->tracing_graph_pause, 0);
3321                         atomic_set(&t->trace_overrun, 0);
3322                         t->curr_ret_stack = -1;
3323                         /* Make sure the tasks see the -1 first: */
3324                         smp_wmb();
3325                         t->ret_stack = ret_stack_list[start++];
3326                 }
3327         } while_each_thread(g, t);
3328
3329 unlock:
3330         read_unlock_irqrestore(&tasklist_lock, flags);
3331 free:
3332         for (i = start; i < end; i++)
3333                 kfree(ret_stack_list[i]);
3334         return ret;
3335 }
3336
3337 static void
3338 ftrace_graph_probe_sched_switch(void *ignore,
3339                         struct task_struct *prev, struct task_struct *next)
3340 {
3341         unsigned long long timestamp;
3342         int index;
3343
3344         /*
3345          * Does the user want to count the time a function was asleep.
3346          * If so, do not update the time stamps.
3347          */
3348         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3349                 return;
3350
3351         timestamp = trace_clock_local();
3352
3353         prev->ftrace_timestamp = timestamp;
3354
3355         /* only process tasks that we timestamped */
3356         if (!next->ftrace_timestamp)
3357                 return;
3358
3359         /*
3360          * Update all the counters in next to make up for the
3361          * time next was sleeping.
3362          */
3363         timestamp -= next->ftrace_timestamp;
3364
3365         for (index = next->curr_ret_stack; index >= 0; index--)
3366                 next->ret_stack[index].calltime += timestamp;
3367 }
3368
3369 /* Allocate a return stack for each task */
3370 static int start_graph_tracing(void)
3371 {
3372         struct ftrace_ret_stack **ret_stack_list;
3373         int ret, cpu;
3374
3375         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3376                                 sizeof(struct ftrace_ret_stack *),
3377                                 GFP_KERNEL);
3378
3379         if (!ret_stack_list)
3380                 return -ENOMEM;
3381
3382         /* The cpu_boot init_task->ret_stack will never be freed */
3383         for_each_online_cpu(cpu) {
3384                 if (!idle_task(cpu)->ret_stack)
3385                         ftrace_graph_init_idle_task(idle_task(cpu), cpu);
3386         }
3387
3388         do {
3389                 ret = alloc_retstack_tasklist(ret_stack_list);
3390         } while (ret == -EAGAIN);
3391
3392         if (!ret) {
3393                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
3394                 if (ret)
3395                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3396                                 " probe to kernel_sched_switch\n");
3397         }
3398
3399         kfree(ret_stack_list);
3400         return ret;
3401 }
3402
3403 /*
3404  * Hibernation protection.
3405  * The state of the current task is too much unstable during
3406  * suspend/restore to disk. We want to protect against that.
3407  */
3408 static int
3409 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3410                                                         void *unused)
3411 {
3412         switch (state) {
3413         case PM_HIBERNATION_PREPARE:
3414                 pause_graph_tracing();
3415                 break;
3416
3417         case PM_POST_HIBERNATION:
3418                 unpause_graph_tracing();
3419                 break;
3420         }
3421         return NOTIFY_DONE;
3422 }
3423
3424 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3425                         trace_func_graph_ent_t entryfunc)
3426 {
3427         int ret = 0;
3428
3429         mutex_lock(&ftrace_lock);
3430
3431         /* we currently allow only one tracer registered at a time */
3432         if (ftrace_graph_active) {
3433                 ret = -EBUSY;
3434                 goto out;
3435         }
3436
3437         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3438         register_pm_notifier(&ftrace_suspend_notifier);
3439
3440         ftrace_graph_active++;
3441         ret = start_graph_tracing();
3442         if (ret) {
3443                 ftrace_graph_active--;
3444                 goto out;
3445         }
3446
3447         ftrace_graph_return = retfunc;
3448         ftrace_graph_entry = entryfunc;
3449
3450         ftrace_startup(FTRACE_START_FUNC_RET);
3451
3452 out:
3453         mutex_unlock(&ftrace_lock);
3454         return ret;
3455 }
3456
3457 void unregister_ftrace_graph(void)
3458 {
3459         mutex_lock(&ftrace_lock);
3460
3461         if (unlikely(!ftrace_graph_active))
3462                 goto out;
3463
3464         ftrace_graph_active--;
3465         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3466         ftrace_graph_entry = ftrace_graph_entry_stub;
3467         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3468         unregister_pm_notifier(&ftrace_suspend_notifier);
3469         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
3470
3471  out:
3472         mutex_unlock(&ftrace_lock);
3473 }
3474
3475 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
3476
3477 static void
3478 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
3479 {
3480         atomic_set(&t->tracing_graph_pause, 0);
3481         atomic_set(&t->trace_overrun, 0);
3482         t->ftrace_timestamp = 0;
3483         /* make curr_ret_stack visible before we add the ret_stack */
3484         smp_wmb();
3485         t->ret_stack = ret_stack;
3486 }
3487
3488 /*
3489  * Allocate a return stack for the idle task. May be the first
3490  * time through, or it may be done by CPU hotplug online.
3491  */
3492 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
3493 {
3494         t->curr_ret_stack = -1;
3495         /*
3496          * The idle task has no parent, it either has its own
3497          * stack or no stack at all.
3498          */
3499         if (t->ret_stack)
3500                 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
3501
3502         if (ftrace_graph_active) {
3503                 struct ftrace_ret_stack *ret_stack;
3504
3505                 ret_stack = per_cpu(idle_ret_stack, cpu);
3506                 if (!ret_stack) {
3507                         ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3508                                             * sizeof(struct ftrace_ret_stack),
3509                                             GFP_KERNEL);
3510                         if (!ret_stack)
3511                                 return;
3512                         per_cpu(idle_ret_stack, cpu) = ret_stack;
3513                 }
3514                 graph_init_task(t, ret_stack);
3515         }
3516 }
3517
3518 /* Allocate a return stack for newly created task */
3519 void ftrace_graph_init_task(struct task_struct *t)
3520 {
3521         /* Make sure we do not use the parent ret_stack */
3522         t->ret_stack = NULL;
3523         t->curr_ret_stack = -1;
3524
3525         if (ftrace_graph_active) {
3526                 struct ftrace_ret_stack *ret_stack;
3527
3528                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3529                                 * sizeof(struct ftrace_ret_stack),
3530                                 GFP_KERNEL);
3531                 if (!ret_stack)
3532                         return;
3533                 graph_init_task(t, ret_stack);
3534         }
3535 }
3536
3537 void ftrace_graph_exit_task(struct task_struct *t)
3538 {
3539         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3540
3541         t->ret_stack = NULL;
3542         /* NULL must become visible to IRQs before we free it: */
3543         barrier();
3544
3545         kfree(ret_stack);
3546 }
3547
3548 void ftrace_graph_stop(void)
3549 {
3550         ftrace_stop();
3551 }
3552 #endif