kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/bsearch.h>
  26 #include <linux/module.h>
  27 #include <linux/ftrace.h>
  28 #include <linux/sysctl.h>
  29 #include <linux/slab.h>
  30 #include <linux/ctype.h>
  31 #include <linux/sort.h>
  32 #include <linux/list.h>
  33 #include <linux/hash.h>
  34 #include <linux/rcupdate.h>
  35
  36 #include <trace/events/sched.h>
  37
  38 #include <asm/setup.h>
  39
  40 #include "trace_output.h"
  41 #include "trace_stat.h"
  42
  43 #define FTRACE_WARN_ON(cond)                    \
  44         ({                                      \
  45                 int ___r = cond;                \
  46                 if (WARN_ON(___r))              \
  47                         ftrace_kill();          \
  48                 ___r;                           \
  49         })
  50
  51 #define FTRACE_WARN_ON_ONCE(cond)               \
  52         ({                                      \
  53                 int ___r = cond;                \
  54                 if (WARN_ON_ONCE(___r))         \
  55                         ftrace_kill();          \
  56                 ___r;                           \
  57         })
  58
  59 /* hash bits for specific function selection */
  60 #define FTRACE_HASH_BITS 7
  61 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  62 #define FTRACE_HASH_DEFAULT_BITS 10
  63 #define FTRACE_HASH_MAX_BITS 12
  64
  65 #define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_CONTROL)
  66
  67 /* ftrace_enabled is a method to turn ftrace on or off */
  68 int ftrace_enabled __read_mostly;
  69 static int last_ftrace_enabled;
  70
  71 /* Quick disabling of function tracer. */
  72 int function_trace_stop;
  73
  74 /* List for set_ftrace_pid's pids. */
  75 LIST_HEAD(ftrace_pids);
  76 struct ftrace_pid {
  77         struct list_head list;
  78         struct pid *pid;
  79 };
  80
  81 /*
  82  * ftrace_disabled is set when an anomaly is discovered.
  83  * ftrace_disabled is much stronger than ftrace_enabled.
  84  */
  85 static int ftrace_disabled __read_mostly;
  86
  87 static DEFINE_MUTEX(ftrace_lock);
  88
  89 static struct ftrace_ops ftrace_list_end __read_mostly = {
  90         .func           = ftrace_stub,
  91 };
  92
  93 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
  94 static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
  95 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
  96 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  97 static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub;
  98 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  99 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
 100 static struct ftrace_ops global_ops;
 101 static struct ftrace_ops control_ops;
 102
 103 static void
 104 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
 105
 106 /*
 107  * Traverse the ftrace_global_list, invoking all entries.  The reason that we
 108  * can use rcu_dereference_raw() is that elements removed from this list
 109  * are simply leaked, so there is no need to interact with a grace-period
 110  * mechanism.  The rcu_dereference_raw() calls are needed to handle
 111  * concurrent insertions into the ftrace_global_list.
 112  *
 113  * Silly Alpha and silly pointer-speculation compiler optimizations!
 114  */
 115 static void ftrace_global_list_func(unsigned long ip,
 116                                     unsigned long parent_ip)
 117 {
 118         struct ftrace_ops *op;
 119
 120         if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
 121                 return;
 122
 123         trace_recursion_set(TRACE_GLOBAL_BIT);
 124         op = rcu_dereference_raw(ftrace_global_list); /*see above*/
 125         while (op != &ftrace_list_end) {
 126                 op->func(ip, parent_ip);
 127                 op = rcu_dereference_raw(op->next); /*see above*/
 128         };
 129         trace_recursion_clear(TRACE_GLOBAL_BIT);
 130 }
 131
 132 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
 133 {
 134         if (!test_tsk_trace_trace(current))
 135                 return;
 136
 137         ftrace_pid_function(ip, parent_ip);
 138 }
 139
 140 static void set_ftrace_pid_function(ftrace_func_t func)
 141 {
 142         /* do not set ftrace_pid_function to itself! */
 143         if (func != ftrace_pid_func)
 144                 ftrace_pid_function = func;
 145 }
 146
 147 /**
 148  * clear_ftrace_function - reset the ftrace function
 149  *
 150  * This NULLs the ftrace function and in essence stops
 151  * tracing.  There may be lag
 152  */
 153 void clear_ftrace_function(void)
 154 {
 155         ftrace_trace_function = ftrace_stub;
 156         __ftrace_trace_function = ftrace_stub;
 157         __ftrace_trace_function_delay = ftrace_stub;
 158         ftrace_pid_function = ftrace_stub;
 159 }
 160
 161 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 162 /*
 163  * For those archs that do not test ftrace_trace_stop in their
 164  * mcount call site, we need to do it from C.
 165  */
 166 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 167 {
 168         if (function_trace_stop)
 169                 return;
 170
 171         __ftrace_trace_function(ip, parent_ip);
 172 }
 173 #endif
 174
 175 static void control_ops_disable_all(struct ftrace_ops *ops)
 176 {
 177         int cpu;
 178
 179         for_each_possible_cpu(cpu)
 180                 *per_cpu_ptr(ops->disabled, cpu) = 1;
 181 }
 182
 183 static int control_ops_alloc(struct ftrace_ops *ops)
 184 {
 185         int __percpu *disabled;
 186
 187         disabled = alloc_percpu(int);
 188         if (!disabled)
 189                 return -ENOMEM;
 190
 191         ops->disabled = disabled;
 192         control_ops_disable_all(ops);
 193         return 0;
 194 }
 195
 196 static void control_ops_free(struct ftrace_ops *ops)
 197 {
 198         free_percpu(ops->disabled);
 199 }
 200
 201 static void update_global_ops(void)
 202 {
 203         ftrace_func_t func;
 204
 205         /*
 206          * If there's only one function registered, then call that
 207          * function directly. Otherwise, we need to iterate over the
 208          * registered callers.
 209          */
 210         if (ftrace_global_list == &ftrace_list_end ||
 211             ftrace_global_list->next == &ftrace_list_end)
 212                 func = ftrace_global_list->func;
 213         else
 214                 func = ftrace_global_list_func;
 215
 216         /* If we filter on pids, update to use the pid function */
 217         if (!list_empty(&ftrace_pids)) {
 218                 set_ftrace_pid_function(func);
 219                 func = ftrace_pid_func;
 220         }
 221
 222         global_ops.func = func;
 223 }
 224
 225 static void update_ftrace_function(void)
 226 {
 227         ftrace_func_t func;
 228
 229         update_global_ops();
 230
 231         /*
 232          * If we are at the end of the list and this ops is
 233          * not dynamic, then have the mcount trampoline call
 234          * the function directly
 235          */
 236         if (ftrace_ops_list == &ftrace_list_end ||
 237             (ftrace_ops_list->next == &ftrace_list_end &&
 238              !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
 239                 func = ftrace_ops_list->func;
 240         else
 241                 func = ftrace_ops_list_func;
 242
 243 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 244         ftrace_trace_function = func;
 245 #else
 246 #ifdef CONFIG_DYNAMIC_FTRACE
 247         /* do not update till all functions have been modified */
 248         __ftrace_trace_function_delay = func;
 249 #else
 250         __ftrace_trace_function = func;
 251 #endif
 252         ftrace_trace_function = ftrace_test_stop_func;
 253 #endif
 254 }
 255
 256 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 257 {
 258         ops->next = *list;
 259         /*
 260          * We are entering ops into the list but another
 261          * CPU might be walking that list. We need to make sure
 262          * the ops->next pointer is valid before another CPU sees
 263          * the ops pointer included into the list.
 264          */
 265         rcu_assign_pointer(*list, ops);
 266 }
 267
 268 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
 269 {
 270         struct ftrace_ops **p;
 271
 272         /*
 273          * If we are removing the last function, then simply point
 274          * to the ftrace_stub.
 275          */
 276         if (*list == ops && ops->next == &ftrace_list_end) {
 277                 *list = &ftrace_list_end;
 278                 return 0;
 279         }
 280
 281         for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
 282                 if (*p == ops)
 283                         break;
 284
 285         if (*p != ops)
 286                 return -1;
 287
 288         *p = (*p)->next;
 289         return 0;
 290 }
 291
 292 static void add_ftrace_list_ops(struct ftrace_ops **list,
 293                                 struct ftrace_ops *main_ops,
 294                                 struct ftrace_ops *ops)
 295 {
 296         int first = *list == &ftrace_list_end;
 297         add_ftrace_ops(list, ops);
 298         if (first)
 299                 add_ftrace_ops(&ftrace_ops_list, main_ops);
 300 }
 301
 302 static int remove_ftrace_list_ops(struct ftrace_ops **list,
 303                                   struct ftrace_ops *main_ops,
 304                                   struct ftrace_ops *ops)
 305 {
 306         int ret = remove_ftrace_ops(list, ops);
 307         if (!ret && *list == &ftrace_list_end)
 308                 ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
 309         return ret;
 310 }
 311
 312 static int __register_ftrace_function(struct ftrace_ops *ops)
 313 {
 314         if (ftrace_disabled)
 315                 return -ENODEV;
 316
 317         if (FTRACE_WARN_ON(ops == &global_ops))
 318                 return -EINVAL;
 319
 320         if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
 321                 return -EBUSY;
 322
 323         /* We don't support both control and global flags set. */
 324         if ((ops->flags & FL_GLOBAL_CONTROL_MASK) == FL_GLOBAL_CONTROL_MASK)
 325                 return -EINVAL;
 326
 327         if (!core_kernel_data((unsigned long)ops))
 328                 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
 329
 330         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 331                 add_ftrace_list_ops(&ftrace_global_list, &global_ops, ops);
 332                 ops->flags |= FTRACE_OPS_FL_ENABLED;
 333         } else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
 334                 if (control_ops_alloc(ops))
 335                         return -ENOMEM;
 336                 add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
 337         } else
 338                 add_ftrace_ops(&ftrace_ops_list, ops);
 339
 340         if (ftrace_enabled)
 341                 update_ftrace_function();
 342
 343         return 0;
 344 }
 345
 346 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 347 {
 348         int ret;
 349
 350         if (ftrace_disabled)
 351                 return -ENODEV;
 352
 353         if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
 354                 return -EBUSY;
 355
 356         if (FTRACE_WARN_ON(ops == &global_ops))
 357                 return -EINVAL;
 358
 359         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
 360                 ret = remove_ftrace_list_ops(&ftrace_global_list,
 361                                              &global_ops, ops);
 362                 if (!ret)
 363                         ops->flags &= ~FTRACE_OPS_FL_ENABLED;
 364         } else if (ops->flags & FTRACE_OPS_FL_CONTROL) {
 365                 ret = remove_ftrace_list_ops(&ftrace_control_list,
 366                                              &control_ops, ops);
 367                 if (!ret) {
 368                         /*
 369                          * The ftrace_ops is now removed from the list,
 370                          * so there'll be no new users. We must ensure
 371                          * all current users are done before we free
 372                          * the control data.
 373                          */
 374                         synchronize_sched();
 375                         control_ops_free(ops);
 376                 }
 377         } else
 378                 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
 379
 380         if (ret < 0)
 381                 return ret;
 382
 383         if (ftrace_enabled)
 384                 update_ftrace_function();
 385
 386         /*
 387          * Dynamic ops may be freed, we must make sure that all
 388          * callers are done before leaving this function.
 389          */
 390         if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
 391                 synchronize_sched();
 392
 393         return 0;
 394 }
 395
 396 static void ftrace_update_pid_func(void)
 397 {
 398         /* Only do something if we are tracing something */
 399         if (ftrace_trace_function == ftrace_stub)
 400                 return;
 401
 402         update_ftrace_function();
 403 }
 404
 405 #ifdef CONFIG_FUNCTION_PROFILER
 406 struct ftrace_profile {
 407         struct hlist_node               node;
 408         unsigned long                   ip;
 409         unsigned long                   counter;
 410 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 411         unsigned long long              time;
 412         unsigned long long              time_squared;
 413 #endif
 414 };
 415
 416 struct ftrace_profile_page {
 417         struct ftrace_profile_page      *next;
 418         unsigned long                   index;
 419         struct ftrace_profile           records[];
 420 };
 421
 422 struct ftrace_profile_stat {
 423         atomic_t                        disabled;
 424         struct hlist_head               *hash;
 425         struct ftrace_profile_page      *pages;
 426         struct ftrace_profile_page      *start;
 427         struct tracer_stat              stat;
 428 };
 429
 430 #define PROFILE_RECORDS_SIZE                                            \
 431         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 432
 433 #define PROFILES_PER_PAGE                                       \
 434         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 435
 436 static int ftrace_profile_bits __read_mostly;
 437 static int ftrace_profile_enabled __read_mostly;
 438
 439 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 440 static DEFINE_MUTEX(ftrace_profile_lock);
 441
 442 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 443
 444 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 445
 446 static void *
 447 function_stat_next(void *v, int idx)
 448 {
 449         struct ftrace_profile *rec = v;
 450         struct ftrace_profile_page *pg;
 451
 452         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 453
 454  again:
 455         if (idx != 0)
 456                 rec++;
 457
 458         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 459                 pg = pg->next;
 460                 if (!pg)
 461                         return NULL;
 462                 rec = &pg->records[0];
 463                 if (!rec->counter)
 464                         goto again;
 465         }
 466
 467         return rec;
 468 }
 469
 470 static void *function_stat_start(struct tracer_stat *trace)
 471 {
 472         struct ftrace_profile_stat *stat =
 473                 container_of(trace, struct ftrace_profile_stat, stat);
 474
 475         if (!stat || !stat->start)
 476                 return NULL;
 477
 478         return function_stat_next(&stat->start->records[0], 0);
 479 }
 480
 481 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 482 /* function graph compares on total time */
 483 static int function_stat_cmp(void *p1, void *p2)
 484 {
 485         struct ftrace_profile *a = p1;
 486         struct ftrace_profile *b = p2;
 487
 488         if (a->time < b->time)
 489                 return -1;
 490         if (a->time > b->time)
 491                 return 1;
 492         else
 493                 return 0;
 494 }
 495 #else
 496 /* not function graph compares against hits */
 497 static int function_stat_cmp(void *p1, void *p2)
 498 {
 499         struct ftrace_profile *a = p1;
 500         struct ftrace_profile *b = p2;
 501
 502         if (a->counter < b->counter)
 503                 return -1;
 504         if (a->counter > b->counter)
 505                 return 1;
 506         else
 507                 return 0;
 508 }
 509 #endif
 510
 511 static int function_stat_headers(struct seq_file *m)
 512 {
 513 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 514         seq_printf(m, "  Function                               "
 515                    "Hit    Time            Avg             s^2\n"
 516                       "  --------                               "
 517                    "---    ----            ---             ---\n");
 518 #else
 519         seq_printf(m, "  Function                               Hit\n"
 520                       "  --------                               ---\n");
 521 #endif
 522         return 0;
 523 }
 524
 525 static int function_stat_show(struct seq_file *m, void *v)
 526 {
 527         struct ftrace_profile *rec = v;
 528         char str[KSYM_SYMBOL_LEN];
 529         int ret = 0;
 530 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 531         static struct trace_seq s;
 532         unsigned long long avg;
 533         unsigned long long stddev;
 534 #endif
 535         mutex_lock(&ftrace_profile_lock);
 536
 537         /* we raced with function_profile_reset() */
 538         if (unlikely(rec->counter == 0)) {
 539                 ret = -EBUSY;
 540                 goto out;
 541         }
 542
 543         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 544         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 545
 546 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 547         seq_printf(m, "    ");
 548         avg = rec->time;
 549         do_div(avg, rec->counter);
 550
 551         /* Sample standard deviation (s^2) */
 552         if (rec->counter <= 1)
 553                 stddev = 0;
 554         else {
 555                 stddev = rec->time_squared - rec->counter * avg * avg;
 556                 /*
 557                  * Divide only 1000 for ns^2 -> us^2 conversion.
 558                  * trace_print_graph_duration will divide 1000 again.
 559                  */
 560                 do_div(stddev, (rec->counter - 1) * 1000);
 561         }
 562
 563         trace_seq_init(&s);
 564         trace_print_graph_duration(rec->time, &s);
 565         trace_seq_puts(&s, "    ");
 566         trace_print_graph_duration(avg, &s);
 567         trace_seq_puts(&s, "    ");
 568         trace_print_graph_duration(stddev, &s);
 569         trace_print_seq(m, &s);
 570 #endif
 571         seq_putc(m, '\n');
 572 out:
 573         mutex_unlock(&ftrace_profile_lock);
 574
 575         return ret;
 576 }
 577
 578 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 579 {
 580         struct ftrace_profile_page *pg;
 581
 582         pg = stat->pages = stat->start;
 583
 584         while (pg) {
 585                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 586                 pg->index = 0;
 587                 pg = pg->next;
 588         }
 589
 590         memset(stat->hash, 0,
 591                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 592 }
 593
 594 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 595 {
 596         struct ftrace_profile_page *pg;
 597         int functions;
 598         int pages;
 599         int i;
 600
 601         /* If we already allocated, do nothing */
 602         if (stat->pages)
 603                 return 0;
 604
 605         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 606         if (!stat->pages)
 607                 return -ENOMEM;
 608
 609 #ifdef CONFIG_DYNAMIC_FTRACE
 610         functions = ftrace_update_tot_cnt;
 611 #else
 612         /*
 613          * We do not know the number of functions that exist because
 614          * dynamic tracing is what counts them. With past experience
 615          * we have around 20K functions. That should be more than enough.
 616          * It is highly unlikely we will execute every function in
 617          * the kernel.
 618          */
 619         functions = 20000;
 620 #endif
 621
 622         pg = stat->start = stat->pages;
 623
 624         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 625
 626         for (i = 0; i < pages; i++) {
 627                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 628                 if (!pg->next)
 629                         goto out_free;
 630                 pg = pg->next;
 631         }
 632
 633         return 0;
 634
 635  out_free:
 636         pg = stat->start;
 637         while (pg) {
 638                 unsigned long tmp = (unsigned long)pg;
 639
 640                 pg = pg->next;
 641                 free_page(tmp);
 642         }
 643
 644         free_page((unsigned long)stat->pages);
 645         stat->pages = NULL;
 646         stat->start = NULL;
 647
 648         return -ENOMEM;
 649 }
 650
 651 static int ftrace_profile_init_cpu(int cpu)
 652 {
 653         struct ftrace_profile_stat *stat;
 654         int size;
 655
 656         stat = &per_cpu(ftrace_profile_stats, cpu);
 657
 658         if (stat->hash) {
 659                 /* If the profile is already created, simply reset it */
 660                 ftrace_profile_reset(stat);
 661                 return 0;
 662         }
 663
 664         /*
 665          * We are profiling all functions, but usually only a few thousand
 666          * functions are hit. We'll make a hash of 1024 items.
 667          */
 668         size = FTRACE_PROFILE_HASH_SIZE;
 669
 670         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 671
 672         if (!stat->hash)
 673                 return -ENOMEM;
 674
 675         if (!ftrace_profile_bits) {
 676                 size--;
 677
 678                 for (; size; size >>= 1)
 679                         ftrace_profile_bits++;
 680         }
 681
 682         /* Preallocate the function profiling pages */
 683         if (ftrace_profile_pages_init(stat) < 0) {
 684                 kfree(stat->hash);
 685                 stat->hash = NULL;
 686                 return -ENOMEM;
 687         }
 688
 689         return 0;
 690 }
 691
 692 static int ftrace_profile_init(void)
 693 {
 694         int cpu;
 695         int ret = 0;
 696
 697         for_each_online_cpu(cpu) {
 698                 ret = ftrace_profile_init_cpu(cpu);
 699                 if (ret)
 700                         break;
 701         }
 702
 703         return ret;
 704 }
 705
 706 /* interrupts must be disabled */
 707 static struct ftrace_profile *
 708 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 709 {
 710         struct ftrace_profile *rec;
 711         struct hlist_head *hhd;
 712         struct hlist_node *n;
 713         unsigned long key;
 714
 715         key = hash_long(ip, ftrace_profile_bits);
 716         hhd = &stat->hash[key];
 717
 718         if (hlist_empty(hhd))
 719                 return NULL;
 720
 721         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 722                 if (rec->ip == ip)
 723                         return rec;
 724         }
 725
 726         return NULL;
 727 }
 728
 729 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 730                                struct ftrace_profile *rec)
 731 {
 732         unsigned long key;
 733
 734         key = hash_long(rec->ip, ftrace_profile_bits);
 735         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 736 }
 737
 738 /*
 739  * The memory is already allocated, this simply finds a new record to use.
 740  */
 741 static struct ftrace_profile *
 742 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 743 {
 744         struct ftrace_profile *rec = NULL;
 745
 746         /* prevent recursion (from NMIs) */
 747         if (atomic_inc_return(&stat->disabled) != 1)
 748                 goto out;
 749
 750         /*
 751          * Try to find the function again since an NMI
 752          * could have added it
 753          */
 754         rec = ftrace_find_profiled_func(stat, ip);
 755         if (rec)
 756                 goto out;
 757
 758         if (stat->pages->index == PROFILES_PER_PAGE) {
 759                 if (!stat->pages->next)
 760                         goto out;
 761                 stat->pages = stat->pages->next;
 762         }
 763
 764         rec = &stat->pages->records[stat->pages->index++];
 765         rec->ip = ip;
 766         ftrace_add_profile(stat, rec);
 767
 768  out:
 769         atomic_dec(&stat->disabled);
 770
 771         return rec;
 772 }
 773
 774 static void
 775 function_profile_call(unsigned long ip, unsigned long parent_ip)
 776 {
 777         struct ftrace_profile_stat *stat;
 778         struct ftrace_profile *rec;
 779         unsigned long flags;
 780
 781         if (!ftrace_profile_enabled)
 782                 return;
 783
 784         local_irq_save(flags);
 785
 786         stat = &__get_cpu_var(ftrace_profile_stats);
 787         if (!stat->hash || !ftrace_profile_enabled)
 788                 goto out;
 789
 790         rec = ftrace_find_profiled_func(stat, ip);
 791         if (!rec) {
 792                 rec = ftrace_profile_alloc(stat, ip);
 793                 if (!rec)
 794                         goto out;
 795         }
 796
 797         rec->counter++;
 798  out:
 799         local_irq_restore(flags);
 800 }
 801
 802 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 803 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 804 {
 805         function_profile_call(trace->func, 0);
 806         return 1;
 807 }
 808
 809 static void profile_graph_return(struct ftrace_graph_ret *trace)
 810 {
 811         struct ftrace_profile_stat *stat;
 812         unsigned long long calltime;
 813         struct ftrace_profile *rec;
 814         unsigned long flags;
 815
 816         local_irq_save(flags);
 817         stat = &__get_cpu_var(ftrace_profile_stats);
 818         if (!stat->hash || !ftrace_profile_enabled)
 819                 goto out;
 820
 821         /* If the calltime was zero'd ignore it */
 822         if (!trace->calltime)
 823                 goto out;
 824
 825         calltime = trace->rettime - trace->calltime;
 826
 827         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 828                 int index;
 829
 830                 index = trace->depth;
 831
 832                 /* Append this call time to the parent time to subtract */
 833                 if (index)
 834                         current->ret_stack[index - 1].subtime += calltime;
 835
 836                 if (current->ret_stack[index].subtime < calltime)
 837                         calltime -= current->ret_stack[index].subtime;
 838                 else
 839                         calltime = 0;
 840         }
 841
 842         rec = ftrace_find_profiled_func(stat, trace->func);
 843         if (rec) {
 844                 rec->time += calltime;
 845                 rec->time_squared += calltime * calltime;
 846         }
 847
 848  out:
 849         local_irq_restore(flags);
 850 }
 851
 852 static int register_ftrace_profiler(void)
 853 {
 854         return register_ftrace_graph(&profile_graph_return,
 855                                      &profile_graph_entry);
 856 }
 857
 858 static void unregister_ftrace_profiler(void)
 859 {
 860         unregister_ftrace_graph();
 861 }
 862 #else
 863 static struct ftrace_ops ftrace_profile_ops __read_mostly = {
 864         .func           = function_profile_call,
 865 };
 866
 867 static int register_ftrace_profiler(void)
 868 {
 869         return register_ftrace_function(&ftrace_profile_ops);
 870 }
 871
 872 static void unregister_ftrace_profiler(void)
 873 {
 874         unregister_ftrace_function(&ftrace_profile_ops);
 875 }
 876 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 877
 878 static ssize_t
 879 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 880                      size_t cnt, loff_t *ppos)
 881 {
 882         unsigned long val;
 883         int ret;
 884
 885         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
 886         if (ret)
 887                 return ret;
 888
 889         val = !!val;
 890
 891         mutex_lock(&ftrace_profile_lock);
 892         if (ftrace_profile_enabled ^ val) {
 893                 if (val) {
 894                         ret = ftrace_profile_init();
 895                         if (ret < 0) {
 896                                 cnt = ret;
 897                                 goto out;
 898                         }
 899
 900                         ret = register_ftrace_profiler();
 901                         if (ret < 0) {
 902                                 cnt = ret;
 903                                 goto out;
 904                         }
 905                         ftrace_profile_enabled = 1;
 906                 } else {
 907                         ftrace_profile_enabled = 0;
 908                         /*
 909                          * unregister_ftrace_profiler calls stop_machine
 910                          * so this acts like an synchronize_sched.
 911                          */
 912                         unregister_ftrace_profiler();
 913                 }
 914         }
 915  out:
 916         mutex_unlock(&ftrace_profile_lock);
 917
 918         *ppos += cnt;
 919
 920         return cnt;
 921 }
 922
 923 static ssize_t
 924 ftrace_profile_read(struct file *filp, char __user *ubuf,
 925                      size_t cnt, loff_t *ppos)
 926 {
 927         char buf[64];           /* big enough to hold a number */
 928         int r;
 929
 930         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 931         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 932 }
 933
 934 static const struct file_operations ftrace_profile_fops = {
 935         .open           = tracing_open_generic,
 936         .read           = ftrace_profile_read,
 937         .write          = ftrace_profile_write,
 938         .llseek         = default_llseek,
 939 };
 940
 941 /* used to initialize the real stat files */
 942 static struct tracer_stat function_stats __initdata = {
 943         .name           = "functions",
 944         .stat_start     = function_stat_start,
 945         .stat_next      = function_stat_next,
 946         .stat_cmp       = function_stat_cmp,
 947         .stat_headers   = function_stat_headers,
 948         .stat_show      = function_stat_show
 949 };
 950
 951 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 952 {
 953         struct ftrace_profile_stat *stat;
 954         struct dentry *entry;
 955         char *name;
 956         int ret;
 957         int cpu;
 958
 959         for_each_possible_cpu(cpu) {
 960                 stat = &per_cpu(ftrace_profile_stats, cpu);
 961
 962                 /* allocate enough for function name + cpu number */
 963                 name = kmalloc(32, GFP_KERNEL);
 964                 if (!name) {
 965                         /*
 966                          * The files created are permanent, if something happens
 967                          * we still do not free memory.
 968                          */
 969                         WARN(1,
 970                              "Could not allocate stat file for cpu %d\n",
 971                              cpu);
 972                         return;
 973                 }
 974                 stat->stat = function_stats;
 975                 snprintf(name, 32, "function%d", cpu);
 976                 stat->stat.name = name;
 977                 ret = register_stat_tracer(&stat->stat);
 978                 if (ret) {
 979                         WARN(1,
 980                              "Could not register function stat for cpu %d\n",
 981                              cpu);
 982                         kfree(name);
 983                         return;
 984                 }
 985         }
 986
 987         entry = debugfs_create_file("function_profile_enabled", 0644,
 988                                     d_tracer, NULL, &ftrace_profile_fops);
 989         if (!entry)
 990                 pr_warning("Could not create debugfs "
 991                            "'function_profile_enabled' entry\n");
 992 }
 993
 994 #else /* CONFIG_FUNCTION_PROFILER */
 995 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 996 {
 997 }
 998 #endif /* CONFIG_FUNCTION_PROFILER */
 999
1000 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
1001
1002 #ifdef CONFIG_DYNAMIC_FTRACE
1003
1004 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
1005 # error Dynamic ftrace depends on MCOUNT_RECORD
1006 #endif
1007
1008 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
1009
1010 struct ftrace_func_probe {
1011         struct hlist_node       node;
1012         struct ftrace_probe_ops *ops;
1013         unsigned long           flags;
1014         unsigned long           ip;
1015         void                    *data;
1016         struct rcu_head         rcu;
1017 };
1018
1019 struct ftrace_func_entry {
1020         struct hlist_node hlist;
1021         unsigned long ip;
1022 };
1023
1024 struct ftrace_hash {
1025         unsigned long           size_bits;
1026         struct hlist_head       *buckets;
1027         unsigned long           count;
1028         struct rcu_head         rcu;
1029 };
1030
1031 /*
1032  * We make these constant because no one should touch them,
1033  * but they are used as the default "empty hash", to avoid allocating
1034  * it all the time. These are in a read only section such that if
1035  * anyone does try to modify it, it will cause an exception.
1036  */
1037 static const struct hlist_head empty_buckets[1];
1038 static const struct ftrace_hash empty_hash = {
1039         .buckets = (struct hlist_head *)empty_buckets,
1040 };
1041 #define EMPTY_HASH      ((struct ftrace_hash *)&empty_hash)
1042
1043 static struct ftrace_ops global_ops = {
1044         .func                   = ftrace_stub,
1045         .notrace_hash           = EMPTY_HASH,
1046         .filter_hash            = EMPTY_HASH,
1047 };
1048
1049 static DEFINE_MUTEX(ftrace_regex_lock);
1050
1051 struct ftrace_page {
1052         struct ftrace_page      *next;
1053         struct dyn_ftrace       *records;
1054         int                     index;
1055         int                     size;
1056 };
1057
1058 static struct ftrace_page *ftrace_new_pgs;
1059
1060 #define ENTRY_SIZE sizeof(struct dyn_ftrace)
1061 #define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
1062
1063 /* estimate from running different kernels */
1064 #define NR_TO_INIT              10000
1065
1066 static struct ftrace_page       *ftrace_pages_start;
1067 static struct ftrace_page       *ftrace_pages;
1068
1069 static bool ftrace_hash_empty(struct ftrace_hash *hash)
1070 {
1071         return !hash || !hash->count;
1072 }
1073
1074 static struct ftrace_func_entry *
1075 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1076 {
1077         unsigned long key;
1078         struct ftrace_func_entry *entry;
1079         struct hlist_head *hhd;
1080         struct hlist_node *n;
1081
1082         if (ftrace_hash_empty(hash))
1083                 return NULL;
1084
1085         if (hash->size_bits > 0)
1086                 key = hash_long(ip, hash->size_bits);
1087         else
1088                 key = 0;
1089
1090         hhd = &hash->buckets[key];
1091
1092         hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1093                 if (entry->ip == ip)
1094                         return entry;
1095         }
1096         return NULL;
1097 }
1098
1099 static void __add_hash_entry(struct ftrace_hash *hash,
1100                              struct ftrace_func_entry *entry)
1101 {
1102         struct hlist_head *hhd;
1103         unsigned long key;
1104
1105         if (hash->size_bits)
1106                 key = hash_long(entry->ip, hash->size_bits);
1107         else
1108                 key = 0;
1109
1110         hhd = &hash->buckets[key];
1111         hlist_add_head(&entry->hlist, hhd);
1112         hash->count++;
1113 }
1114
1115 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1116 {
1117         struct ftrace_func_entry *entry;
1118
1119         entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1120         if (!entry)
1121                 return -ENOMEM;
1122
1123         entry->ip = ip;
1124         __add_hash_entry(hash, entry);
1125
1126         return 0;
1127 }
1128
1129 static void
1130 free_hash_entry(struct ftrace_hash *hash,
1131                   struct ftrace_func_entry *entry)
1132 {
1133         hlist_del(&entry->hlist);
1134         kfree(entry);
1135         hash->count--;
1136 }
1137
1138 static void
1139 remove_hash_entry(struct ftrace_hash *hash,
1140                   struct ftrace_func_entry *entry)
1141 {
1142         hlist_del(&entry->hlist);
1143         hash->count--;
1144 }
1145
1146 static void ftrace_hash_clear(struct ftrace_hash *hash)
1147 {
1148         struct hlist_head *hhd;
1149         struct hlist_node *tp, *tn;
1150         struct ftrace_func_entry *entry;
1151         int size = 1 << hash->size_bits;
1152         int i;
1153
1154         if (!hash->count)
1155                 return;
1156
1157         for (i = 0; i < size; i++) {
1158                 hhd = &hash->buckets[i];
1159                 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1160                         free_hash_entry(hash, entry);
1161         }
1162         FTRACE_WARN_ON(hash->count);
1163 }
1164
1165 static void free_ftrace_hash(struct ftrace_hash *hash)
1166 {
1167         if (!hash || hash == EMPTY_HASH)
1168                 return;
1169         ftrace_hash_clear(hash);
1170         kfree(hash->buckets);
1171         kfree(hash);
1172 }
1173
1174 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1175 {
1176         struct ftrace_hash *hash;
1177
1178         hash = container_of(rcu, struct ftrace_hash, rcu);
1179         free_ftrace_hash(hash);
1180 }
1181
1182 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1183 {
1184         if (!hash || hash == EMPTY_HASH)
1185                 return;
1186         call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1187 }
1188
1189 void ftrace_free_filter(struct ftrace_ops *ops)
1190 {
1191         free_ftrace_hash(ops->filter_hash);
1192         free_ftrace_hash(ops->notrace_hash);
1193 }
1194
1195 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1196 {
1197         struct ftrace_hash *hash;
1198         int size;
1199
1200         hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1201         if (!hash)
1202                 return NULL;
1203
1204         size = 1 << size_bits;
1205         hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
1206
1207         if (!hash->buckets) {
1208                 kfree(hash);
1209                 return NULL;
1210         }
1211
1212         hash->size_bits = size_bits;
1213
1214         return hash;
1215 }
1216
1217 static struct ftrace_hash *
1218 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1219 {
1220         struct ftrace_func_entry *entry;
1221         struct ftrace_hash *new_hash;
1222         struct hlist_node *tp;
1223         int size;
1224         int ret;
1225         int i;
1226
1227         new_hash = alloc_ftrace_hash(size_bits);
1228         if (!new_hash)
1229                 return NULL;
1230
1231         /* Empty hash? */
1232         if (ftrace_hash_empty(hash))
1233                 return new_hash;
1234
1235         size = 1 << hash->size_bits;
1236         for (i = 0; i < size; i++) {
1237                 hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1238                         ret = add_hash_entry(new_hash, entry->ip);
1239                         if (ret < 0)
1240                                 goto free_hash;
1241                 }
1242         }
1243
1244         FTRACE_WARN_ON(new_hash->count != hash->count);
1245
1246         return new_hash;
1247
1248  free_hash:
1249         free_ftrace_hash(new_hash);
1250         return NULL;
1251 }
1252
1253 static void
1254 ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1255 static void
1256 ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1257
1258 static int
1259 ftrace_hash_move(struct ftrace_ops *ops, int enable,
1260                  struct ftrace_hash **dst, struct ftrace_hash *src)
1261 {
1262         struct ftrace_func_entry *entry;
1263         struct hlist_node *tp, *tn;
1264         struct hlist_head *hhd;
1265         struct ftrace_hash *old_hash;
1266         struct ftrace_hash *new_hash;
1267         unsigned long key;
1268         int size = src->count;
1269         int bits = 0;
1270         int ret;
1271         int i;
1272
1273         /*
1274          * Remove the current set, update the hash and add
1275          * them back.
1276          */
1277         ftrace_hash_rec_disable(ops, enable);
1278
1279         /*
1280          * If the new source is empty, just free dst and assign it
1281          * the empty_hash.
1282          */
1283         if (!src->count) {
1284                 free_ftrace_hash_rcu(*dst);
1285                 rcu_assign_pointer(*dst, EMPTY_HASH);
1286                 /* still need to update the function records */
1287                 ret = 0;
1288                 goto out;
1289         }
1290
1291         /*
1292          * Make the hash size about 1/2 the # found
1293          */
1294         for (size /= 2; size; size >>= 1)
1295                 bits++;
1296
1297         /* Don't allocate too much */
1298         if (bits > FTRACE_HASH_MAX_BITS)
1299                 bits = FTRACE_HASH_MAX_BITS;
1300
1301         ret = -ENOMEM;
1302         new_hash = alloc_ftrace_hash(bits);
1303         if (!new_hash)
1304                 goto out;
1305
1306         size = 1 << src->size_bits;
1307         for (i = 0; i < size; i++) {
1308                 hhd = &src->buckets[i];
1309                 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1310                         if (bits > 0)
1311                                 key = hash_long(entry->ip, bits);
1312                         else
1313                                 key = 0;
1314                         remove_hash_entry(src, entry);
1315                         __add_hash_entry(new_hash, entry);
1316                 }
1317         }
1318
1319         old_hash = *dst;
1320         rcu_assign_pointer(*dst, new_hash);
1321         free_ftrace_hash_rcu(old_hash);
1322
1323         ret = 0;
1324  out:
1325         /*
1326          * Enable regardless of ret:
1327          *  On success, we enable the new hash.
1328          *  On failure, we re-enable the original hash.
1329          */
1330         ftrace_hash_rec_enable(ops, enable);
1331
1332         return ret;
1333 }
1334
1335 /*
1336  * Test the hashes for this ops to see if we want to call
1337  * the ops->func or not.
1338  *
1339  * It's a match if the ip is in the ops->filter_hash or
1340  * the filter_hash does not exist or is empty,
1341  *  AND
1342  * the ip is not in the ops->notrace_hash.
1343  *
1344  * This needs to be called with preemption disabled as
1345  * the hashes are freed with call_rcu_sched().
1346  */
1347 static int
1348 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1349 {
1350         struct ftrace_hash *filter_hash;
1351         struct ftrace_hash *notrace_hash;
1352         int ret;
1353
1354         filter_hash = rcu_dereference_raw(ops->filter_hash);
1355         notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1356
1357         if ((ftrace_hash_empty(filter_hash) ||
1358              ftrace_lookup_ip(filter_hash, ip)) &&
1359             (ftrace_hash_empty(notrace_hash) ||
1360              !ftrace_lookup_ip(notrace_hash, ip)))
1361                 ret = 1;
1362         else
1363                 ret = 0;
1364
1365         return ret;
1366 }
1367
1368 /*
1369  * This is a double for. Do not use 'break' to break out of the loop,
1370  * you must use a goto.
1371  */
1372 #define do_for_each_ftrace_rec(pg, rec)                                 \
1373         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
1374                 int _____i;                                             \
1375                 for (_____i = 0; _____i < pg->index; _____i++) {        \
1376                         rec = &pg->records[_____i];
1377
1378 #define while_for_each_ftrace_rec()             \
1379                 }                               \
1380         }
1381
1382
1383 static int ftrace_cmp_recs(const void *a, const void *b)
1384 {
1385         const struct dyn_ftrace *reca = a;
1386         const struct dyn_ftrace *recb = b;
1387
1388         if (reca->ip > recb->ip)
1389                 return 1;
1390         if (reca->ip < recb->ip)
1391                 return -1;
1392         return 0;
1393 }
1394
1395 /**
1396  * ftrace_location - return true if the ip giving is a traced location
1397  * @ip: the instruction pointer to check
1398  *
1399  * Returns 1 if @ip given is a pointer to a ftrace location.
1400  * That is, the instruction that is either a NOP or call to
1401  * the function tracer. It checks the ftrace internal tables to
1402  * determine if the address belongs or not.
1403  */
1404 int ftrace_location(unsigned long ip)
1405 {
1406         struct ftrace_page *pg;
1407         struct dyn_ftrace *rec;
1408         struct dyn_ftrace key;
1409
1410         key.ip = ip;
1411
1412         for (pg = ftrace_pages_start; pg; pg = pg->next) {
1413                 rec = bsearch(&key, pg->records, pg->index,
1414                               sizeof(struct dyn_ftrace),
1415                               ftrace_cmp_recs);
1416                 if (rec)
1417                         return 1;
1418         }
1419
1420         return 0;
1421 }
1422
1423 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1424                                      int filter_hash,
1425                                      bool inc)
1426 {
1427         struct ftrace_hash *hash;
1428         struct ftrace_hash *other_hash;
1429         struct ftrace_page *pg;
1430         struct dyn_ftrace *rec;
1431         int count = 0;
1432         int all = 0;
1433
1434         /* Only update if the ops has been registered */
1435         if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1436                 return;
1437
1438         /*
1439          * In the filter_hash case:
1440          *   If the count is zero, we update all records.
1441          *   Otherwise we just update the items in the hash.
1442          *
1443          * In the notrace_hash case:
1444          *   We enable the update in the hash.
1445          *   As disabling notrace means enabling the tracing,
1446          *   and enabling notrace means disabling, the inc variable
1447          *   gets inversed.
1448          */
1449         if (filter_hash) {
1450                 hash = ops->filter_hash;
1451                 other_hash = ops->notrace_hash;
1452                 if (ftrace_hash_empty(hash))
1453                         all = 1;
1454         } else {
1455                 inc = !inc;
1456                 hash = ops->notrace_hash;
1457                 other_hash = ops->filter_hash;
1458                 /*
1459                  * If the notrace hash has no items,
1460                  * then there's nothing to do.
1461                  */
1462                 if (ftrace_hash_empty(hash))
1463                         return;
1464         }
1465
1466         do_for_each_ftrace_rec(pg, rec) {
1467                 int in_other_hash = 0;
1468                 int in_hash = 0;
1469                 int match = 0;
1470
1471                 if (all) {
1472                         /*
1473                          * Only the filter_hash affects all records.
1474                          * Update if the record is not in the notrace hash.
1475                          */
1476                         if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1477                                 match = 1;
1478                 } else {
1479                         in_hash = !!ftrace_lookup_ip(hash, rec->ip);
1480                         in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
1481
1482                         /*
1483                          *
1484                          */
1485                         if (filter_hash && in_hash && !in_other_hash)
1486                                 match = 1;
1487                         else if (!filter_hash && in_hash &&
1488                                  (in_other_hash || ftrace_hash_empty(other_hash)))
1489                                 match = 1;
1490                 }
1491                 if (!match)
1492                         continue;
1493
1494                 if (inc) {
1495                         rec->flags++;
1496                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1497                                 return;
1498                 } else {
1499                         if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1500                                 return;
1501                         rec->flags--;
1502                 }
1503                 count++;
1504                 /* Shortcut, if we handled all records, we are done. */
1505                 if (!all && count == hash->count)
1506                         return;
1507         } while_for_each_ftrace_rec();
1508 }
1509
1510 static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1511                                     int filter_hash)
1512 {
1513         __ftrace_hash_rec_update(ops, filter_hash, 0);
1514 }
1515
1516 static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1517                                    int filter_hash)
1518 {
1519         __ftrace_hash_rec_update(ops, filter_hash, 1);
1520 }
1521
1522 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
1523 {
1524         if (ftrace_pages->index == ftrace_pages->size) {
1525                 /* We should have allocated enough */
1526                 if (WARN_ON(!ftrace_pages->next))
1527                         return NULL;
1528                 ftrace_pages = ftrace_pages->next;
1529         }
1530
1531         return &ftrace_pages->records[ftrace_pages->index++];
1532 }
1533
1534 static struct dyn_ftrace *
1535 ftrace_record_ip(unsigned long ip)
1536 {
1537         struct dyn_ftrace *rec;
1538
1539         if (ftrace_disabled)
1540                 return NULL;
1541
1542         rec = ftrace_alloc_dyn_node(ip);
1543         if (!rec)
1544                 return NULL;
1545
1546         rec->ip = ip;
1547
1548         return rec;
1549 }
1550
1551 static void print_ip_ins(const char *fmt, unsigned char *p)
1552 {
1553         int i;
1554
1555         printk(KERN_CONT "%s", fmt);
1556
1557         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1558                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1559 }
1560
1561 /**
1562  * ftrace_bug - report and shutdown function tracer
1563  * @failed: The failed type (EFAULT, EINVAL, EPERM)
1564  * @ip: The address that failed
1565  *
1566  * The arch code that enables or disables the function tracing
1567  * can call ftrace_bug() when it has detected a problem in
1568  * modifying the code. @failed should be one of either:
1569  * EFAULT - if the problem happens on reading the @ip address
1570  * EINVAL - if what is read at @ip is not what was expected
1571  * EPERM - if the problem happens on writting to the @ip address
1572  */
1573 void ftrace_bug(int failed, unsigned long ip)
1574 {
1575         switch (failed) {
1576         case -EFAULT:
1577                 FTRACE_WARN_ON_ONCE(1);
1578                 pr_info("ftrace faulted on modifying ");
1579                 print_ip_sym(ip);
1580                 break;
1581         case -EINVAL:
1582                 FTRACE_WARN_ON_ONCE(1);
1583                 pr_info("ftrace failed to modify ");
1584                 print_ip_sym(ip);
1585                 print_ip_ins(" actual: ", (unsigned char *)ip);
1586                 printk(KERN_CONT "\n");
1587                 break;
1588         case -EPERM:
1589                 FTRACE_WARN_ON_ONCE(1);
1590                 pr_info("ftrace faulted on writing ");
1591                 print_ip_sym(ip);
1592                 break;
1593         default:
1594                 FTRACE_WARN_ON_ONCE(1);
1595                 pr_info("ftrace faulted on unknown error ");
1596                 print_ip_sym(ip);
1597         }
1598 }
1599
1600
1601 /* Return 1 if the address range is reserved for ftrace */
1602 int ftrace_text_reserved(void *start, void *end)
1603 {
1604         struct dyn_ftrace *rec;
1605         struct ftrace_page *pg;
1606
1607         do_for_each_ftrace_rec(pg, rec) {
1608                 if (rec->ip <= (unsigned long)end &&
1609                     rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1610                         return 1;
1611         } while_for_each_ftrace_rec();
1612         return 0;
1613 }
1614
1615 static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
1616 {
1617         unsigned long flag = 0UL;
1618
1619         /*
1620          * If we are updating calls:
1621          *
1622          *   If the record has a ref count, then we need to enable it
1623          *   because someone is using it.
1624          *
1625          *   Otherwise we make sure its disabled.
1626          *
1627          * If we are disabling calls, then disable all records that
1628          * are enabled.
1629          */
1630         if (enable && (rec->flags & ~FTRACE_FL_MASK))
1631                 flag = FTRACE_FL_ENABLED;
1632
1633         /* If the state of this record hasn't changed, then do nothing */
1634         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1635                 return FTRACE_UPDATE_IGNORE;
1636
1637         if (flag) {
1638                 if (update)
1639                         rec->flags |= FTRACE_FL_ENABLED;
1640                 return FTRACE_UPDATE_MAKE_CALL;
1641         }
1642
1643         if (update)
1644                 rec->flags &= ~FTRACE_FL_ENABLED;
1645
1646         return FTRACE_UPDATE_MAKE_NOP;
1647 }
1648
1649 /**
1650  * ftrace_update_record, set a record that now is tracing or not
1651  * @rec: the record to update
1652  * @enable: set to 1 if the record is tracing, zero to force disable
1653  *
1654  * The records that represent all functions that can be traced need
1655  * to be updated when tracing has been enabled.
1656  */
1657 int ftrace_update_record(struct dyn_ftrace *rec, int enable)
1658 {
1659         return ftrace_check_record(rec, enable, 1);
1660 }
1661
1662 /**
1663  * ftrace_test_record, check if the record has been enabled or not
1664  * @rec: the record to test
1665  * @enable: set to 1 to check if enabled, 0 if it is disabled
1666  *
1667  * The arch code may need to test if a record is already set to
1668  * tracing to determine how to modify the function code that it
1669  * represents.
1670  */
1671 int ftrace_test_record(struct dyn_ftrace *rec, int enable)
1672 {
1673         return ftrace_check_record(rec, enable, 0);
1674 }
1675
1676 static int
1677 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1678 {
1679         unsigned long ftrace_addr;
1680         int ret;
1681
1682         ftrace_addr = (unsigned long)FTRACE_ADDR;
1683
1684         ret = ftrace_update_record(rec, enable);
1685
1686         switch (ret) {
1687         case FTRACE_UPDATE_IGNORE:
1688                 return 0;
1689
1690         case FTRACE_UPDATE_MAKE_CALL:
1691                 return ftrace_make_call(rec, ftrace_addr);
1692
1693         case FTRACE_UPDATE_MAKE_NOP:
1694                 return ftrace_make_nop(NULL, rec, ftrace_addr);
1695         }
1696
1697         return -1; /* unknow ftrace bug */
1698 }
1699
1700 static void ftrace_replace_code(int update)
1701 {
1702         struct dyn_ftrace *rec;
1703         struct ftrace_page *pg;
1704         int failed;
1705
1706         if (unlikely(ftrace_disabled))
1707                 return;
1708
1709         do_for_each_ftrace_rec(pg, rec) {
1710                 failed = __ftrace_replace_code(rec, update);
1711                 if (failed) {
1712                         ftrace_bug(failed, rec->ip);
1713                         /* Stop processing */
1714                         return;
1715                 }
1716         } while_for_each_ftrace_rec();
1717 }
1718
1719 struct ftrace_rec_iter {
1720         struct ftrace_page      *pg;
1721         int                     index;
1722 };
1723
1724 /**
1725  * ftrace_rec_iter_start, start up iterating over traced functions
1726  *
1727  * Returns an iterator handle that is used to iterate over all
1728  * the records that represent address locations where functions
1729  * are traced.
1730  *
1731  * May return NULL if no records are available.
1732  */
1733 struct ftrace_rec_iter *ftrace_rec_iter_start(void)
1734 {
1735         /*
1736          * We only use a single iterator.
1737          * Protected by the ftrace_lock mutex.
1738          */
1739         static struct ftrace_rec_iter ftrace_rec_iter;
1740         struct ftrace_rec_iter *iter = &ftrace_rec_iter;
1741
1742         iter->pg = ftrace_pages_start;
1743         iter->index = 0;
1744
1745         /* Could have empty pages */
1746         while (iter->pg && !iter->pg->index)
1747                 iter->pg = iter->pg->next;
1748
1749         if (!iter->pg)
1750                 return NULL;
1751
1752         return iter;
1753 }
1754
1755 /**
1756  * ftrace_rec_iter_next, get the next record to process.
1757  * @iter: The handle to the iterator.
1758  *
1759  * Returns the next iterator after the given iterator @iter.
1760  */
1761 struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
1762 {
1763         iter->index++;
1764
1765         if (iter->index >= iter->pg->index) {
1766                 iter->pg = iter->pg->next;
1767                 iter->index = 0;
1768
1769                 /* Could have empty pages */
1770                 while (iter->pg && !iter->pg->index)
1771                         iter->pg = iter->pg->next;
1772         }
1773
1774         if (!iter->pg)
1775                 return NULL;
1776
1777         return iter;
1778 }
1779
1780 /**
1781  * ftrace_rec_iter_record, get the record at the iterator location
1782  * @iter: The current iterator location
1783  *
1784  * Returns the record that the current @iter is at.
1785  */
1786 struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
1787 {
1788         return &iter->pg->records[iter->index];
1789 }
1790
1791 static int
1792 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1793 {
1794         unsigned long ip;
1795         int ret;
1796
1797         ip = rec->ip;
1798
1799         if (unlikely(ftrace_disabled))
1800                 return 0;
1801
1802         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1803         if (ret) {
1804                 ftrace_bug(ret, ip);
1805                 return 0;
1806         }
1807         return 1;
1808 }
1809
1810 /*
1811  * archs can override this function if they must do something
1812  * before the modifying code is performed.
1813  */
1814 int __weak ftrace_arch_code_modify_prepare(void)
1815 {
1816         return 0;
1817 }
1818
1819 /*
1820  * archs can override this function if they must do something
1821  * after the modifying code is performed.
1822  */
1823 int __weak ftrace_arch_code_modify_post_process(void)
1824 {
1825         return 0;
1826 }
1827
1828 static int __ftrace_modify_code(void *data)
1829 {
1830         int *command = data;
1831
1832         if (*command & FTRACE_UPDATE_CALLS)
1833                 ftrace_replace_code(1);
1834         else if (*command & FTRACE_DISABLE_CALLS)
1835                 ftrace_replace_code(0);
1836
1837         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1838                 ftrace_update_ftrace_func(ftrace_trace_function);
1839
1840         if (*command & FTRACE_START_FUNC_RET)
1841                 ftrace_enable_ftrace_graph_caller();
1842         else if (*command & FTRACE_STOP_FUNC_RET)
1843                 ftrace_disable_ftrace_graph_caller();
1844
1845         return 0;
1846 }
1847
1848 /**
1849  * ftrace_run_stop_machine, go back to the stop machine method
1850  * @command: The command to tell ftrace what to do
1851  *
1852  * If an arch needs to fall back to the stop machine method, the
1853  * it can call this function.
1854  */
1855 void ftrace_run_stop_machine(int command)
1856 {
1857         stop_machine(__ftrace_modify_code, &command, NULL);
1858 }
1859
1860 /**
1861  * arch_ftrace_update_code, modify the code to trace or not trace
1862  * @command: The command that needs to be done
1863  *
1864  * Archs can override this function if it does not need to
1865  * run stop_machine() to modify code.
1866  */
1867 void __weak arch_ftrace_update_code(int command)
1868 {
1869         ftrace_run_stop_machine(command);
1870 }
1871
1872 static void ftrace_run_update_code(int command)
1873 {
1874         int ret;
1875
1876         ret = ftrace_arch_code_modify_prepare();
1877         FTRACE_WARN_ON(ret);
1878         if (ret)
1879                 return;
1880         /*
1881          * Do not call function tracer while we update the code.
1882          * We are in stop machine.
1883          */
1884         function_trace_stop++;
1885
1886         /*
1887          * By default we use stop_machine() to modify the code.
1888          * But archs can do what ever they want as long as it
1889          * is safe. The stop_machine() is the safest, but also
1890          * produces the most overhead.
1891          */
1892         arch_ftrace_update_code(command);
1893
1894 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
1895         /*
1896          * For archs that call ftrace_test_stop_func(), we must
1897          * wait till after we update all the function callers
1898          * before we update the callback. This keeps different
1899          * ops that record different functions from corrupting
1900          * each other.
1901          */
1902         __ftrace_trace_function = __ftrace_trace_function_delay;
1903 #endif
1904         function_trace_stop--;
1905
1906         ret = ftrace_arch_code_modify_post_process();
1907         FTRACE_WARN_ON(ret);
1908 }
1909
1910 static ftrace_func_t saved_ftrace_func;
1911 static int ftrace_start_up;
1912 static int global_start_up;
1913
1914 static void ftrace_startup_enable(int command)
1915 {
1916         if (saved_ftrace_func != ftrace_trace_function) {
1917                 saved_ftrace_func = ftrace_trace_function;
1918                 command |= FTRACE_UPDATE_TRACE_FUNC;
1919         }
1920
1921         if (!command || !ftrace_enabled)
1922                 return;
1923
1924         ftrace_run_update_code(command);
1925 }
1926
1927 static int ftrace_startup(struct ftrace_ops *ops, int command)
1928 {
1929         bool hash_enable = true;
1930
1931         if (unlikely(ftrace_disabled))
1932                 return -ENODEV;
1933
1934         ftrace_start_up++;
1935         command |= FTRACE_UPDATE_CALLS;
1936
1937         /* ops marked global share the filter hashes */
1938         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1939                 ops = &global_ops;
1940                 /* Don't update hash if global is already set */
1941                 if (global_start_up)
1942                         hash_enable = false;
1943                 global_start_up++;
1944         }
1945
1946         ops->flags |= FTRACE_OPS_FL_ENABLED;
1947         if (hash_enable)
1948                 ftrace_hash_rec_enable(ops, 1);
1949
1950         ftrace_startup_enable(command);
1951
1952         return 0;
1953 }
1954
1955 static void ftrace_shutdown(struct ftrace_ops *ops, int command)
1956 {
1957         bool hash_disable = true;
1958
1959         if (unlikely(ftrace_disabled))
1960                 return;
1961
1962         ftrace_start_up--;
1963         /*
1964          * Just warn in case of unbalance, no need to kill ftrace, it's not
1965          * critical but the ftrace_call callers may be never nopped again after
1966          * further ftrace uses.
1967          */
1968         WARN_ON_ONCE(ftrace_start_up < 0);
1969
1970         if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1971                 ops = &global_ops;
1972                 global_start_up--;
1973                 WARN_ON_ONCE(global_start_up < 0);
1974                 /* Don't update hash if global still has users */
1975                 if (global_start_up) {
1976                         WARN_ON_ONCE(!ftrace_start_up);
1977                         hash_disable = false;
1978                 }
1979         }
1980
1981         if (hash_disable)
1982                 ftrace_hash_rec_disable(ops, 1);
1983
1984         if (ops != &global_ops || !global_start_up)
1985                 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
1986
1987         command |= FTRACE_UPDATE_CALLS;
1988
1989         if (saved_ftrace_func != ftrace_trace_function) {
1990                 saved_ftrace_func = ftrace_trace_function;
1991                 command |= FTRACE_UPDATE_TRACE_FUNC;
1992         }
1993
1994         if (!command || !ftrace_enabled)
1995                 return;
1996
1997         ftrace_run_update_code(command);
1998 }
1999
2000 static void ftrace_startup_sysctl(void)
2001 {
2002         if (unlikely(ftrace_disabled))
2003                 return;
2004
2005         /* Force update next time */
2006         saved_ftrace_func = NULL;
2007         /* ftrace_start_up is true if we want ftrace running */
2008         if (ftrace_start_up)
2009                 ftrace_run_update_code(FTRACE_UPDATE_CALLS);
2010 }
2011
2012 static void ftrace_shutdown_sysctl(void)
2013 {
2014         if (unlikely(ftrace_disabled))
2015                 return;
2016
2017         /* ftrace_start_up is true if ftrace is running */
2018         if (ftrace_start_up)
2019                 ftrace_run_update_code(FTRACE_DISABLE_CALLS);
2020 }
2021
2022 static cycle_t          ftrace_update_time;
2023 static unsigned long    ftrace_update_cnt;
2024 unsigned long           ftrace_update_tot_cnt;
2025
2026 static int ops_traces_mod(struct ftrace_ops *ops)
2027 {
2028         struct ftrace_hash *hash;
2029
2030         hash = ops->filter_hash;
2031         return ftrace_hash_empty(hash);
2032 }
2033
2034 static int ftrace_update_code(struct module *mod)
2035 {
2036         struct ftrace_page *pg;
2037         struct dyn_ftrace *p;
2038         cycle_t start, stop;
2039         unsigned long ref = 0;
2040         int i;
2041
2042         /*
2043          * When adding a module, we need to check if tracers are
2044          * currently enabled and if they are set to trace all functions.
2045          * If they are, we need to enable the module functions as well
2046          * as update the reference counts for those function records.
2047          */
2048         if (mod) {
2049                 struct ftrace_ops *ops;
2050
2051                 for (ops = ftrace_ops_list;
2052                      ops != &ftrace_list_end; ops = ops->next) {
2053                         if (ops->flags & FTRACE_OPS_FL_ENABLED &&
2054                             ops_traces_mod(ops))
2055                                 ref++;
2056                 }
2057         }
2058
2059         start = ftrace_now(raw_smp_processor_id());
2060         ftrace_update_cnt = 0;
2061
2062         for (pg = ftrace_new_pgs; pg; pg = pg->next) {
2063
2064                 for (i = 0; i < pg->index; i++) {
2065                         /* If something went wrong, bail without enabling anything */
2066                         if (unlikely(ftrace_disabled))
2067                                 return -1;
2068
2069                         p = &pg->records[i];
2070                         p->flags = ref;
2071
2072                         /*
2073                          * Do the initial record conversion from mcount jump
2074                          * to the NOP instructions.
2075                          */
2076                         if (!ftrace_code_disable(mod, p))
2077                                 break;
2078
2079                         ftrace_update_cnt++;
2080
2081                         /*
2082                          * If the tracing is enabled, go ahead and enable the record.
2083                          *
2084                          * The reason not to enable the record immediatelly is the
2085                          * inherent check of ftrace_make_nop/ftrace_make_call for
2086                          * correct previous instructions.  Making first the NOP
2087                          * conversion puts the module to the correct state, thus
2088                          * passing the ftrace_make_call check.
2089                          */
2090                         if (ftrace_start_up && ref) {
2091                                 int failed = __ftrace_replace_code(p, 1);
2092                                 if (failed)
2093                                         ftrace_bug(failed, p->ip);
2094                         }
2095                 }
2096         }
2097
2098         ftrace_new_pgs = NULL;
2099
2100         stop = ftrace_now(raw_smp_processor_id());
2101         ftrace_update_time = stop - start;
2102         ftrace_update_tot_cnt += ftrace_update_cnt;
2103
2104         return 0;
2105 }
2106
2107 static int ftrace_allocate_records(struct ftrace_page *pg, int count)
2108 {
2109         int order;
2110         int cnt;
2111
2112         if (WARN_ON(!count))
2113                 return -EINVAL;
2114
2115         order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
2116
2117         /*
2118          * We want to fill as much as possible. No more than a page
2119          * may be empty.
2120          */
2121         while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
2122                 order--;
2123
2124  again:
2125         pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
2126
2127         if (!pg->records) {
2128                 /* if we can't allocate this size, try something smaller */
2129                 if (!order)
2130                         return -ENOMEM;
2131                 order >>= 1;
2132                 goto again;
2133         }
2134
2135         cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
2136         pg->size = cnt;
2137
2138         if (cnt > count)
2139                 cnt = count;
2140
2141         return cnt;
2142 }
2143
2144 static struct ftrace_page *
2145 ftrace_allocate_pages(unsigned long num_to_init)
2146 {
2147         struct ftrace_page *start_pg;
2148         struct ftrace_page *pg;
2149         int order;
2150         int cnt;
2151
2152         if (!num_to_init)
2153                 return 0;
2154
2155         start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
2156         if (!pg)
2157                 return NULL;
2158
2159         /*
2160          * Try to allocate as much as possible in one continues
2161          * location that fills in all of the space. We want to
2162          * waste as little space as possible.
2163          */
2164         for (;;) {
2165                 cnt = ftrace_allocate_records(pg, num_to_init);
2166                 if (cnt < 0)
2167                         goto free_pages;
2168
2169                 num_to_init -= cnt;
2170                 if (!num_to_init)
2171                         break;
2172
2173                 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
2174                 if (!pg->next)
2175                         goto free_pages;
2176
2177                 pg = pg->next;
2178         }
2179
2180         return start_pg;
2181
2182  free_pages:
2183         while (start_pg) {
2184                 order = get_count_order(pg->size / ENTRIES_PER_PAGE);
2185                 free_pages((unsigned long)pg->records, order);
2186                 start_pg = pg->next;
2187                 kfree(pg);
2188                 pg = start_pg;
2189         }
2190         pr_info("ftrace: FAILED to allocate memory for functions\n");
2191         return NULL;
2192 }
2193
2194 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
2195 {
2196         int cnt;
2197
2198         if (!num_to_init) {
2199                 pr_info("ftrace: No functions to be traced?\n");
2200                 return -1;
2201         }
2202
2203         cnt = num_to_init / ENTRIES_PER_PAGE;
2204         pr_info("ftrace: allocating %ld entries in %d pages\n",
2205                 num_to_init, cnt + 1);
2206
2207         return 0;
2208 }
2209
2210 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
2211
2212 struct ftrace_iterator {
2213         loff_t                          pos;
2214         loff_t                          func_pos;
2215         struct ftrace_page              *pg;
2216         struct dyn_ftrace               *func;
2217         struct ftrace_func_probe        *probe;
2218         struct trace_parser             parser;
2219         struct ftrace_hash              *hash;
2220         struct ftrace_ops               *ops;
2221         int                             hidx;
2222         int                             idx;
2223         unsigned                        flags;
2224 };
2225
2226 static void *
2227 t_hash_next(struct seq_file *m, loff_t *pos)
2228 {
2229         struct ftrace_iterator *iter = m->private;
2230         struct hlist_node *hnd = NULL;
2231         struct hlist_head *hhd;
2232
2233         (*pos)++;
2234         iter->pos = *pos;
2235
2236         if (iter->probe)
2237                 hnd = &iter->probe->node;
2238  retry:
2239         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
2240                 return NULL;
2241
2242         hhd = &ftrace_func_hash[iter->hidx];
2243
2244         if (hlist_empty(hhd)) {
2245                 iter->hidx++;
2246                 hnd = NULL;
2247                 goto retry;
2248         }
2249
2250         if (!hnd)
2251                 hnd = hhd->first;
2252         else {
2253                 hnd = hnd->next;
2254                 if (!hnd) {
2255                         iter->hidx++;
2256                         goto retry;
2257                 }
2258         }
2259
2260         if (WARN_ON_ONCE(!hnd))
2261                 return NULL;
2262
2263         iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
2264
2265         return iter;
2266 }
2267
2268 static void *t_hash_start(struct seq_file *m, loff_t *pos)
2269 {
2270         struct ftrace_iterator *iter = m->private;
2271         void *p = NULL;
2272         loff_t l;
2273
2274         if (!(iter->flags & FTRACE_ITER_DO_HASH))
2275                 return NULL;
2276
2277         if (iter->func_pos > *pos)
2278                 return NULL;
2279
2280         iter->hidx = 0;
2281         for (l = 0; l <= (*pos - iter->func_pos); ) {
2282                 p = t_hash_next(m, &l);
2283                 if (!p)
2284                         break;
2285         }
2286         if (!p)
2287                 return NULL;
2288
2289         /* Only set this if we have an item */
2290         iter->flags |= FTRACE_ITER_HASH;
2291
2292         return iter;
2293 }
2294
2295 static int
2296 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
2297 {
2298         struct ftrace_func_probe *rec;
2299
2300         rec = iter->probe;
2301         if (WARN_ON_ONCE(!rec))
2302                 return -EIO;
2303
2304         if (rec->ops->print)
2305                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2306
2307         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
2308
2309         if (rec->data)
2310                 seq_printf(m, ":%p", rec->data);
2311         seq_putc(m, '\n');
2312
2313         return 0;
2314 }
2315
2316 static void *
2317 t_next(struct seq_file *m, void *v, loff_t *pos)
2318 {
2319         struct ftrace_iterator *iter = m->private;
2320         struct ftrace_ops *ops = iter->ops;
2321         struct dyn_ftrace *rec = NULL;
2322
2323         if (unlikely(ftrace_disabled))
2324                 return NULL;
2325
2326         if (iter->flags & FTRACE_ITER_HASH)
2327                 return t_hash_next(m, pos);
2328
2329         (*pos)++;
2330         iter->pos = iter->func_pos = *pos;
2331
2332         if (iter->flags & FTRACE_ITER_PRINTALL)
2333                 return t_hash_start(m, pos);
2334
2335  retry:
2336         if (iter->idx >= iter->pg->index) {
2337                 if (iter->pg->next) {
2338                         iter->pg = iter->pg->next;
2339                         iter->idx = 0;
2340                         goto retry;
2341                 }
2342         } else {
2343                 rec = &iter->pg->records[iter->idx++];
2344                 if (((iter->flags & FTRACE_ITER_FILTER) &&
2345                      !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
2346
2347                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
2348                      !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2349
2350                     ((iter->flags & FTRACE_ITER_ENABLED) &&
2351                      !(rec->flags & ~FTRACE_FL_MASK))) {
2352
2353                         rec = NULL;
2354                         goto retry;
2355                 }
2356         }
2357
2358         if (!rec)
2359                 return t_hash_start(m, pos);
2360
2361         iter->func = rec;
2362
2363         return iter;
2364 }
2365
2366 static void reset_iter_read(struct ftrace_iterator *iter)
2367 {
2368         iter->pos = 0;
2369         iter->func_pos = 0;
2370         iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
2371 }
2372
2373 static void *t_start(struct seq_file *m, loff_t *pos)
2374 {
2375         struct ftrace_iterator *iter = m->private;
2376         struct ftrace_ops *ops = iter->ops;
2377         void *p = NULL;
2378         loff_t l;
2379
2380         mutex_lock(&ftrace_lock);
2381
2382         if (unlikely(ftrace_disabled))
2383                 return NULL;
2384
2385         /*
2386          * If an lseek was done, then reset and start from beginning.
2387          */
2388         if (*pos < iter->pos)
2389                 reset_iter_read(iter);
2390
2391         /*
2392          * For set_ftrace_filter reading, if we have the filter
2393          * off, we can short cut and just print out that all
2394          * functions are enabled.
2395          */
2396         if (iter->flags & FTRACE_ITER_FILTER &&
2397             ftrace_hash_empty(ops->filter_hash)) {
2398                 if (*pos > 0)
2399                         return t_hash_start(m, pos);
2400                 iter->flags |= FTRACE_ITER_PRINTALL;
2401                 /* reset in case of seek/pread */
2402                 iter->flags &= ~FTRACE_ITER_HASH;
2403                 return iter;
2404         }
2405
2406         if (iter->flags & FTRACE_ITER_HASH)
2407                 return t_hash_start(m, pos);
2408
2409         /*
2410          * Unfortunately, we need to restart at ftrace_pages_start
2411          * every time we let go of the ftrace_mutex. This is because
2412          * those pointers can change without the lock.
2413          */
2414         iter->pg = ftrace_pages_start;
2415         iter->idx = 0;
2416         for (l = 0; l <= *pos; ) {
2417                 p = t_next(m, p, &l);
2418                 if (!p)
2419                         break;
2420         }
2421
2422         if (!p)
2423                 return t_hash_start(m, pos);
2424
2425         return iter;
2426 }
2427
2428 static void t_stop(struct seq_file *m, void *p)
2429 {
2430         mutex_unlock(&ftrace_lock);
2431 }
2432
2433 static int t_show(struct seq_file *m, void *v)
2434 {
2435         struct ftrace_iterator *iter = m->private;
2436         struct dyn_ftrace *rec;
2437
2438         if (iter->flags & FTRACE_ITER_HASH)
2439                 return t_hash_show(m, iter);
2440
2441         if (iter->flags & FTRACE_ITER_PRINTALL) {
2442                 seq_printf(m, "#### all functions enabled ####\n");
2443                 return 0;
2444         }
2445
2446         rec = iter->func;
2447
2448         if (!rec)
2449                 return 0;
2450
2451         seq_printf(m, "%ps", (void *)rec->ip);
2452         if (iter->flags & FTRACE_ITER_ENABLED)
2453                 seq_printf(m, " (%ld)",
2454                            rec->flags & ~FTRACE_FL_MASK);
2455         seq_printf(m, "\n");
2456
2457         return 0;
2458 }
2459
2460 static const struct seq_operations show_ftrace_seq_ops = {
2461         .start = t_start,
2462         .next = t_next,
2463         .stop = t_stop,
2464         .show = t_show,
2465 };
2466
2467 static int
2468 ftrace_avail_open(struct inode *inode, struct file *file)
2469 {
2470         struct ftrace_iterator *iter;
2471         int ret;
2472
2473         if (unlikely(ftrace_disabled))
2474                 return -ENODEV;
2475
2476         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2477         if (!iter)
2478                 return -ENOMEM;
2479
2480         iter->pg = ftrace_pages_start;
2481         iter->ops = &global_ops;
2482
2483         ret = seq_open(file, &show_ftrace_seq_ops);
2484         if (!ret) {
2485                 struct seq_file *m = file->private_data;
2486
2487                 m->private = iter;
2488         } else {
2489                 kfree(iter);
2490         }
2491
2492         return ret;
2493 }
2494
2495 static int
2496 ftrace_enabled_open(struct inode *inode, struct file *file)
2497 {
2498         struct ftrace_iterator *iter;
2499         int ret;
2500
2501         if (unlikely(ftrace_disabled))
2502                 return -ENODEV;
2503
2504         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2505         if (!iter)
2506                 return -ENOMEM;
2507
2508         iter->pg = ftrace_pages_start;
2509         iter->flags = FTRACE_ITER_ENABLED;
2510         iter->ops = &global_ops;
2511
2512         ret = seq_open(file, &show_ftrace_seq_ops);
2513         if (!ret) {
2514                 struct seq_file *m = file->private_data;
2515
2516                 m->private = iter;
2517         } else {
2518                 kfree(iter);
2519         }
2520
2521         return ret;
2522 }
2523
2524 static void ftrace_filter_reset(struct ftrace_hash *hash)
2525 {
2526         mutex_lock(&ftrace_lock);
2527         ftrace_hash_clear(hash);
2528         mutex_unlock(&ftrace_lock);
2529 }
2530
2531 /**
2532  * ftrace_regex_open - initialize function tracer filter files
2533  * @ops: The ftrace_ops that hold the hash filters
2534  * @flag: The type of filter to process
2535  * @inode: The inode, usually passed in to your open routine
2536  * @file: The file, usually passed in to your open routine
2537  *
2538  * ftrace_regex_open() initializes the filter files for the
2539  * @ops. Depending on @flag it may process the filter hash or
2540  * the notrace hash of @ops. With this called from the open
2541  * routine, you can use ftrace_filter_write() for the write
2542  * routine if @flag has FTRACE_ITER_FILTER set, or
2543  * ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
2544  * ftrace_regex_lseek() should be used as the lseek routine, and
2545  * release must call ftrace_regex_release().
2546  */
2547 int
2548 ftrace_regex_open(struct ftrace_ops *ops, int flag,
2549                   struct inode *inode, struct file *file)
2550 {
2551         struct ftrace_iterator *iter;
2552         struct ftrace_hash *hash;
2553         int ret = 0;
2554
2555         if (unlikely(ftrace_disabled))
2556                 return -ENODEV;
2557
2558         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2559         if (!iter)
2560                 return -ENOMEM;
2561
2562         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2563                 kfree(iter);
2564                 return -ENOMEM;
2565         }
2566
2567         if (flag & FTRACE_ITER_NOTRACE)
2568                 hash = ops->notrace_hash;
2569         else
2570                 hash = ops->filter_hash;
2571
2572         iter->ops = ops;
2573         iter->flags = flag;
2574
2575         if (file->f_mode & FMODE_WRITE) {
2576                 mutex_lock(&ftrace_lock);
2577                 iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2578                 mutex_unlock(&ftrace_lock);
2579
2580                 if (!iter->hash) {
2581                         trace_parser_put(&iter->parser);
2582                         kfree(iter);
2583                         return -ENOMEM;
2584                 }
2585         }
2586
2587         mutex_lock(&ftrace_regex_lock);
2588
2589         if ((file->f_mode & FMODE_WRITE) &&
2590             (file->f_flags & O_TRUNC))
2591                 ftrace_filter_reset(iter->hash);
2592
2593         if (file->f_mode & FMODE_READ) {
2594                 iter->pg = ftrace_pages_start;
2595
2596                 ret = seq_open(file, &show_ftrace_seq_ops);
2597                 if (!ret) {
2598                         struct seq_file *m = file->private_data;
2599                         m->private = iter;
2600                 } else {
2601                         /* Failed */
2602                         free_ftrace_hash(iter->hash);
2603                         trace_parser_put(&iter->parser);
2604                         kfree(iter);
2605                 }
2606         } else
2607                 file->private_data = iter;
2608         mutex_unlock(&ftrace_regex_lock);
2609
2610         return ret;
2611 }
2612
2613 static int
2614 ftrace_filter_open(struct inode *inode, struct file *file)
2615 {
2616         return ftrace_regex_open(&global_ops,
2617                         FTRACE_ITER_FILTER | FTRACE_ITER_DO_HASH,
2618                         inode, file);
2619 }
2620
2621 static int
2622 ftrace_notrace_open(struct inode *inode, struct file *file)
2623 {
2624         return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2625                                  inode, file);
2626 }
2627
2628 loff_t
2629 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
2630 {
2631         loff_t ret;
2632
2633         if (file->f_mode & FMODE_READ)
2634                 ret = seq_lseek(file, offset, origin);
2635         else
2636                 file->f_pos = ret = 1;
2637
2638         return ret;
2639 }
2640
2641 static int ftrace_match(char *str, char *regex, int len, int type)
2642 {
2643         int matched = 0;
2644         int slen;
2645
2646         switch (type) {
2647         case MATCH_FULL:
2648                 if (strcmp(str, regex) == 0)
2649                         matched = 1;
2650                 break;
2651         case MATCH_FRONT_ONLY:
2652                 if (strncmp(str, regex, len) == 0)
2653                         matched = 1;
2654                 break;
2655         case MATCH_MIDDLE_ONLY:
2656                 if (strstr(str, regex))
2657                         matched = 1;
2658                 break;
2659         case MATCH_END_ONLY:
2660                 slen = strlen(str);
2661                 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2662                         matched = 1;
2663                 break;
2664         }
2665
2666         return matched;
2667 }
2668
2669 static int
2670 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2671 {
2672         struct ftrace_func_entry *entry;
2673         int ret = 0;
2674
2675         entry = ftrace_lookup_ip(hash, rec->ip);
2676         if (not) {
2677                 /* Do nothing if it doesn't exist */
2678                 if (!entry)
2679                         return 0;
2680
2681                 free_hash_entry(hash, entry);
2682         } else {
2683                 /* Do nothing if it exists */
2684                 if (entry)
2685                         return 0;
2686
2687                 ret = add_hash_entry(hash, rec->ip);
2688         }
2689         return ret;
2690 }
2691
2692 static int
2693 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2694                     char *regex, int len, int type)
2695 {
2696         char str[KSYM_SYMBOL_LEN];
2697         char *modname;
2698
2699         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2700
2701         if (mod) {
2702                 /* module lookup requires matching the module */
2703                 if (!modname || strcmp(modname, mod))
2704                         return 0;
2705
2706                 /* blank search means to match all funcs in the mod */
2707                 if (!len)
2708                         return 1;
2709         }
2710
2711         return ftrace_match(str, regex, len, type);
2712 }
2713
2714 static int
2715 match_records(struct ftrace_hash *hash, char *buff,
2716               int len, char *mod, int not)
2717 {
2718         unsigned search_len = 0;
2719         struct ftrace_page *pg;
2720         struct dyn_ftrace *rec;
2721         int type = MATCH_FULL;
2722         char *search = buff;
2723         int found = 0;
2724         int ret;
2725
2726         if (len) {
2727                 type = filter_parse_regex(buff, len, &search, &not);
2728                 search_len = strlen(search);
2729         }
2730
2731         mutex_lock(&ftrace_lock);
2732
2733         if (unlikely(ftrace_disabled))
2734                 goto out_unlock;
2735
2736         do_for_each_ftrace_rec(pg, rec) {
2737                 if (ftrace_match_record(rec, mod, search, search_len, type)) {
2738                         ret = enter_record(hash, rec, not);
2739                         if (ret < 0) {
2740                                 found = ret;
2741                                 goto out_unlock;
2742                         }
2743                         found = 1;
2744                 }
2745         } while_for_each_ftrace_rec();
2746  out_unlock:
2747         mutex_unlock(&ftrace_lock);
2748
2749         return found;
2750 }
2751
2752 static int
2753 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2754 {
2755         return match_records(hash, buff, len, NULL, 0);
2756 }
2757
2758 static int
2759 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2760 {
2761         int not = 0;
2762
2763         /* blank or '*' mean the same */
2764         if (strcmp(buff, "*") == 0)
2765                 buff[0] = 0;
2766
2767         /* handle the case of 'dont filter this module' */
2768         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
2769                 buff[0] = 0;
2770                 not = 1;
2771         }
2772
2773         return match_records(hash, buff, strlen(buff), mod, not);
2774 }
2775
2776 /*
2777  * We register the module command as a template to show others how
2778  * to register the a command as well.
2779  */
2780
2781 static int
2782 ftrace_mod_callback(struct ftrace_hash *hash,
2783                     char *func, char *cmd, char *param, int enable)
2784 {
2785         char *mod;
2786         int ret = -EINVAL;
2787
2788         /*
2789          * cmd == 'mod' because we only registered this func
2790          * for the 'mod' ftrace_func_command.
2791          * But if you register one func with multiple commands,
2792          * you can tell which command was used by the cmd
2793          * parameter.
2794          */
2795
2796         /* we must have a module name */
2797         if (!param)
2798                 return ret;
2799
2800         mod = strsep(&param, ":");
2801         if (!strlen(mod))
2802                 return ret;
2803
2804         ret = ftrace_match_module_records(hash, func, mod);
2805         if (!ret)
2806                 ret = -EINVAL;
2807         if (ret < 0)
2808                 return ret;
2809
2810         return 0;
2811 }
2812
2813 static struct ftrace_func_command ftrace_mod_cmd = {
2814         .name                   = "mod",
2815         .func                   = ftrace_mod_callback,
2816 };
2817
2818 static int __init ftrace_mod_cmd_init(void)
2819 {
2820         return register_ftrace_command(&ftrace_mod_cmd);
2821 }
2822 device_initcall(ftrace_mod_cmd_init);
2823
2824 static void
2825 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
2826 {
2827         struct ftrace_func_probe *entry;
2828         struct hlist_head *hhd;
2829         struct hlist_node *n;
2830         unsigned long key;
2831
2832         key = hash_long(ip, FTRACE_HASH_BITS);
2833
2834         hhd = &ftrace_func_hash[key];
2835
2836         if (hlist_empty(hhd))
2837                 return;
2838
2839         /*
2840          * Disable preemption for these calls to prevent a RCU grace
2841          * period. This syncs the hash iteration and freeing of items
2842          * on the hash. rcu_read_lock is too dangerous here.
2843          */
2844         preempt_disable_notrace();
2845         hlist_for_each_entry_rcu(entry, n, hhd, node) {
2846                 if (entry->ip == ip)
2847                         entry->ops->func(ip, parent_ip, &entry->data);
2848         }
2849         preempt_enable_notrace();
2850 }
2851
2852 static struct ftrace_ops trace_probe_ops __read_mostly =
2853 {
2854         .func           = function_trace_probe_call,
2855 };
2856
2857 static int ftrace_probe_registered;
2858
2859 static void __enable_ftrace_function_probe(void)
2860 {
2861         int ret;
2862         int i;
2863
2864         if (ftrace_probe_registered)
2865                 return;
2866
2867         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2868                 struct hlist_head *hhd = &ftrace_func_hash[i];
2869                 if (hhd->first)
2870                         break;
2871         }
2872         /* Nothing registered? */
2873         if (i == FTRACE_FUNC_HASHSIZE)
2874                 return;
2875
2876         ret = __register_ftrace_function(&trace_probe_ops);
2877         if (!ret)
2878                 ret = ftrace_startup(&trace_probe_ops, 0);
2879
2880         ftrace_probe_registered = 1;
2881 }
2882
2883 static void __disable_ftrace_function_probe(void)
2884 {
2885         int ret;
2886         int i;
2887
2888         if (!ftrace_probe_registered)
2889                 return;
2890
2891         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2892                 struct hlist_head *hhd = &ftrace_func_hash[i];
2893                 if (hhd->first)
2894                         return;
2895         }
2896
2897         /* no more funcs left */
2898         ret = __unregister_ftrace_function(&trace_probe_ops);
2899         if (!ret)
2900                 ftrace_shutdown(&trace_probe_ops, 0);
2901
2902         ftrace_probe_registered = 0;
2903 }
2904
2905
2906 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2907 {
2908         struct ftrace_func_probe *entry =
2909                 container_of(rhp, struct ftrace_func_probe, rcu);
2910
2911         if (entry->ops->free)
2912                 entry->ops->free(&entry->data);
2913         kfree(entry);
2914 }
2915
2916
2917 int
2918 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2919                               void *data)
2920 {
2921         struct ftrace_func_probe *entry;
2922         struct ftrace_page *pg;
2923         struct dyn_ftrace *rec;
2924         int type, len, not;
2925         unsigned long key;
2926         int count = 0;
2927         char *search;
2928
2929         type = filter_parse_regex(glob, strlen(glob), &search, &not);
2930         len = strlen(search);
2931
2932         /* we do not support '!' for function probes */
2933         if (WARN_ON(not))
2934                 return -EINVAL;
2935
2936         mutex_lock(&ftrace_lock);
2937
2938         if (unlikely(ftrace_disabled))
2939                 goto out_unlock;
2940
2941         do_for_each_ftrace_rec(pg, rec) {
2942
2943                 if (!ftrace_match_record(rec, NULL, search, len, type))
2944                         continue;
2945
2946                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2947                 if (!entry) {
2948                         /* If we did not process any, then return error */
2949                         if (!count)
2950                                 count = -ENOMEM;
2951                         goto out_unlock;
2952                 }
2953
2954                 count++;
2955
2956                 entry->data = data;
2957
2958                 /*
2959                  * The caller might want to do something special
2960                  * for each function we find. We call the callback
2961                  * to give the caller an opportunity to do so.
2962                  */
2963                 if (ops->callback) {
2964                         if (ops->callback(rec->ip, &entry->data) < 0) {
2965                                 /* caller does not like this func */
2966                                 kfree(entry);
2967                                 continue;
2968                         }
2969                 }
2970
2971                 entry->ops = ops;
2972                 entry->ip = rec->ip;
2973
2974                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2975                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2976
2977         } while_for_each_ftrace_rec();
2978         __enable_ftrace_function_probe();
2979
2980  out_unlock:
2981         mutex_unlock(&ftrace_lock);
2982
2983         return count;
2984 }
2985
2986 enum {
2987         PROBE_TEST_FUNC         = 1,
2988         PROBE_TEST_DATA         = 2
2989 };
2990
2991 static void
2992 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2993                                   void *data, int flags)
2994 {
2995         struct ftrace_func_probe *entry;
2996         struct hlist_node *n, *tmp;
2997         char str[KSYM_SYMBOL_LEN];
2998         int type = MATCH_FULL;
2999         int i, len = 0;
3000         char *search;
3001
3002         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
3003                 glob = NULL;
3004         else if (glob) {
3005                 int not;
3006
3007                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
3008                 len = strlen(search);
3009
3010                 /* we do not support '!' for function probes */
3011                 if (WARN_ON(not))
3012                         return;
3013         }
3014
3015         mutex_lock(&ftrace_lock);
3016         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
3017                 struct hlist_head *hhd = &ftrace_func_hash[i];
3018
3019                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
3020
3021                         /* break up if statements for readability */
3022                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
3023                                 continue;
3024
3025                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
3026                                 continue;
3027
3028                         /* do this last, since it is the most expensive */
3029                         if (glob) {
3030                                 kallsyms_lookup(entry->ip, NULL, NULL,
3031                                                 NULL, str);
3032                                 if (!ftrace_match(str, glob, len, type))
3033                                         continue;
3034                         }
3035
3036                         hlist_del(&entry->node);
3037                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
3038                 }
3039         }
3040         __disable_ftrace_function_probe();
3041         mutex_unlock(&ftrace_lock);
3042 }
3043
3044 void
3045 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
3046                                 void *data)
3047 {
3048         __unregister_ftrace_function_probe(glob, ops, data,
3049                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
3050 }
3051
3052 void
3053 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
3054 {
3055         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
3056 }
3057
3058 void unregister_ftrace_function_probe_all(char *glob)
3059 {
3060         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
3061 }
3062
3063 static LIST_HEAD(ftrace_commands);
3064 static DEFINE_MUTEX(ftrace_cmd_mutex);
3065
3066 int register_ftrace_command(struct ftrace_func_command *cmd)
3067 {
3068         struct ftrace_func_command *p;
3069         int ret = 0;
3070
3071         mutex_lock(&ftrace_cmd_mutex);
3072         list_for_each_entry(p, &ftrace_commands, list) {
3073                 if (strcmp(cmd->name, p->name) == 0) {
3074                         ret = -EBUSY;
3075                         goto out_unlock;
3076                 }
3077         }
3078         list_add(&cmd->list, &ftrace_commands);
3079  out_unlock:
3080         mutex_unlock(&ftrace_cmd_mutex);
3081
3082         return ret;
3083 }
3084
3085 int unregister_ftrace_command(struct ftrace_func_command *cmd)
3086 {
3087         struct ftrace_func_command *p, *n;
3088         int ret = -ENODEV;
3089
3090         mutex_lock(&ftrace_cmd_mutex);
3091         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
3092                 if (strcmp(cmd->name, p->name) == 0) {
3093                         ret = 0;
3094                         list_del_init(&p->list);
3095                         goto out_unlock;
3096                 }
3097         }
3098  out_unlock:
3099         mutex_unlock(&ftrace_cmd_mutex);
3100
3101         return ret;
3102 }
3103
3104 static int ftrace_process_regex(struct ftrace_hash *hash,
3105                                 char *buff, int len, int enable)
3106 {
3107         char *func, *command, *next = buff;
3108         struct ftrace_func_command *p;
3109         int ret = -EINVAL;
3110
3111         func = strsep(&next, ":");
3112
3113         if (!next) {
3114                 ret = ftrace_match_records(hash, func, len);
3115                 if (!ret)
3116                         ret = -EINVAL;
3117                 if (ret < 0)
3118                         return ret;
3119                 return 0;
3120         }
3121
3122         /* command found */
3123
3124         command = strsep(&next, ":");
3125
3126         mutex_lock(&ftrace_cmd_mutex);
3127         list_for_each_entry(p, &ftrace_commands, list) {
3128                 if (strcmp(p->name, command) == 0) {
3129                         ret = p->func(hash, func, command, next, enable);
3130                         goto out_unlock;
3131                 }
3132         }
3133  out_unlock:
3134         mutex_unlock(&ftrace_cmd_mutex);
3135
3136         return ret;
3137 }
3138
3139 static ssize_t
3140 ftrace_regex_write(struct file *file, const char __user *ubuf,
3141                    size_t cnt, loff_t *ppos, int enable)
3142 {
3143         struct ftrace_iterator *iter;
3144         struct trace_parser *parser;
3145         ssize_t ret, read;
3146
3147         if (!cnt)
3148                 return 0;
3149
3150         mutex_lock(&ftrace_regex_lock);
3151
3152         ret = -ENODEV;
3153         if (unlikely(ftrace_disabled))
3154                 goto out_unlock;
3155
3156         if (file->f_mode & FMODE_READ) {
3157                 struct seq_file *m = file->private_data;
3158                 iter = m->private;
3159         } else
3160                 iter = file->private_data;
3161
3162         parser = &iter->parser;
3163         read = trace_get_user(parser, ubuf, cnt, ppos);
3164
3165         if (read >= 0 && trace_parser_loaded(parser) &&
3166             !trace_parser_cont(parser)) {
3167                 ret = ftrace_process_regex(iter->hash, parser->buffer,
3168                                            parser->idx, enable);
3169                 trace_parser_clear(parser);
3170                 if (ret)
3171                         goto out_unlock;
3172         }
3173
3174         ret = read;
3175 out_unlock:
3176         mutex_unlock(&ftrace_regex_lock);
3177
3178         return ret;
3179 }
3180
3181 ssize_t
3182 ftrace_filter_write(struct file *file, const char __user *ubuf,
3183                     size_t cnt, loff_t *ppos)
3184 {
3185         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
3186 }
3187
3188 ssize_t
3189 ftrace_notrace_write(struct file *file, const char __user *ubuf,
3190                      size_t cnt, loff_t *ppos)
3191 {
3192         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
3193 }
3194
3195 static int
3196 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
3197                  int reset, int enable)
3198 {
3199         struct ftrace_hash **orig_hash;
3200         struct ftrace_hash *hash;
3201         int ret;
3202
3203         /* All global ops uses the global ops filters */
3204         if (ops->flags & FTRACE_OPS_FL_GLOBAL)
3205                 ops = &global_ops;
3206
3207         if (unlikely(ftrace_disabled))
3208                 return -ENODEV;
3209
3210         if (enable)
3211                 orig_hash = &ops->filter_hash;
3212         else
3213                 orig_hash = &ops->notrace_hash;
3214
3215         hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
3216         if (!hash)
3217                 return -ENOMEM;
3218
3219         mutex_lock(&ftrace_regex_lock);
3220         if (reset)
3221                 ftrace_filter_reset(hash);
3222         if (buf && !ftrace_match_records(hash, buf, len)) {
3223                 ret = -EINVAL;
3224                 goto out_regex_unlock;
3225         }
3226
3227         mutex_lock(&ftrace_lock);
3228         ret = ftrace_hash_move(ops, enable, orig_hash, hash);
3229         if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
3230             && ftrace_enabled)
3231                 ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3232
3233         mutex_unlock(&ftrace_lock);
3234
3235  out_regex_unlock:
3236         mutex_unlock(&ftrace_regex_lock);
3237
3238         free_ftrace_hash(hash);
3239         return ret;
3240 }
3241
3242 /**
3243  * ftrace_set_filter - set a function to filter on in ftrace
3244  * @ops - the ops to set the filter with
3245  * @buf - the string that holds the function filter text.
3246  * @len - the length of the string.
3247  * @reset - non zero to reset all filters before applying this filter.
3248  *
3249  * Filters denote which functions should be enabled when tracing is enabled.
3250  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3251  */
3252 int ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
3253                        int len, int reset)
3254 {
3255         return ftrace_set_regex(ops, buf, len, reset, 1);
3256 }
3257 EXPORT_SYMBOL_GPL(ftrace_set_filter);
3258
3259 /**
3260  * ftrace_set_notrace - set a function to not trace in ftrace
3261  * @ops - the ops to set the notrace filter with
3262  * @buf - the string that holds the function notrace text.
3263  * @len - the length of the string.
3264  * @reset - non zero to reset all filters before applying this filter.
3265  *
3266  * Notrace Filters denote which functions should not be enabled when tracing
3267  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3268  * for tracing.
3269  */
3270 int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
3271                         int len, int reset)
3272 {
3273         return ftrace_set_regex(ops, buf, len, reset, 0);
3274 }
3275 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
3276 /**
3277  * ftrace_set_filter - set a function to filter on in ftrace
3278  * @ops - the ops to set the filter with
3279  * @buf - the string that holds the function filter text.
3280  * @len - the length of the string.
3281  * @reset - non zero to reset all filters before applying this filter.
3282  *
3283  * Filters denote which functions should be enabled when tracing is enabled.
3284  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
3285  */
3286 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
3287 {
3288         ftrace_set_regex(&global_ops, buf, len, reset, 1);
3289 }
3290 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
3291
3292 /**
3293  * ftrace_set_notrace - set a function to not trace in ftrace
3294  * @ops - the ops to set the notrace filter with
3295  * @buf - the string that holds the function notrace text.
3296  * @len - the length of the string.
3297  * @reset - non zero to reset all filters before applying this filter.
3298  *
3299  * Notrace Filters denote which functions should not be enabled when tracing
3300  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
3301  * for tracing.
3302  */
3303 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
3304 {
3305         ftrace_set_regex(&global_ops, buf, len, reset, 0);
3306 }
3307 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
3308
3309 /*
3310  * command line interface to allow users to set filters on boot up.
3311  */
3312 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
3313 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3314 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3315
3316 static int __init set_ftrace_notrace(char *str)
3317 {
3318         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3319         return 1;
3320 }
3321 __setup("ftrace_notrace=", set_ftrace_notrace);
3322
3323 static int __init set_ftrace_filter(char *str)
3324 {
3325         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3326         return 1;
3327 }
3328 __setup("ftrace_filter=", set_ftrace_filter);
3329
3330 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3331 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
3332 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
3333
3334 static int __init set_graph_function(char *str)
3335 {
3336         strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
3337         return 1;
3338 }
3339 __setup("ftrace_graph_filter=", set_graph_function);
3340
3341 static void __init set_ftrace_early_graph(char *buf)
3342 {
3343         int ret;
3344         char *func;
3345
3346         while (buf) {
3347                 func = strsep(&buf, ",");
3348                 /* we allow only one expression at a time */
3349                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3350                                       func);
3351                 if (ret)
3352                         printk(KERN_DEBUG "ftrace: function %s not "
3353                                           "traceable\n", func);
3354         }
3355 }
3356 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3357
3358 void __init
3359 ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3360 {
3361         char *func;
3362
3363         while (buf) {
3364                 func = strsep(&buf, ",");
3365                 ftrace_set_regex(ops, func, strlen(func), 0, enable);
3366         }
3367 }
3368
3369 static void __init set_ftrace_early_filters(void)
3370 {
3371         if (ftrace_filter_buf[0])
3372                 ftrace_set_early_filter(&global_ops, ftrace_filter_buf, 1);
3373         if (ftrace_notrace_buf[0])
3374                 ftrace_set_early_filter(&global_ops, ftrace_notrace_buf, 0);
3375 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3376         if (ftrace_graph_buf[0])
3377                 set_ftrace_early_graph(ftrace_graph_buf);
3378 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3379 }
3380
3381 int ftrace_regex_release(struct inode *inode, struct file *file)
3382 {
3383         struct seq_file *m = (struct seq_file *)file->private_data;
3384         struct ftrace_iterator *iter;
3385         struct ftrace_hash **orig_hash;
3386         struct trace_parser *parser;
3387         int filter_hash;
3388         int ret;
3389
3390         mutex_lock(&ftrace_regex_lock);
3391         if (file->f_mode & FMODE_READ) {
3392                 iter = m->private;
3393
3394                 seq_release(inode, file);
3395         } else
3396                 iter = file->private_data;
3397
3398         parser = &iter->parser;
3399         if (trace_parser_loaded(parser)) {
3400                 parser->buffer[parser->idx] = 0;
3401                 ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3402         }
3403
3404         trace_parser_put(parser);
3405
3406         if (file->f_mode & FMODE_WRITE) {
3407                 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3408
3409                 if (filter_hash)
3410                         orig_hash = &iter->ops->filter_hash;
3411                 else
3412                         orig_hash = &iter->ops->notrace_hash;
3413
3414                 mutex_lock(&ftrace_lock);
3415                 ret = ftrace_hash_move(iter->ops, filter_hash,
3416                                        orig_hash, iter->hash);
3417                 if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
3418                     && ftrace_enabled)
3419                         ftrace_run_update_code(FTRACE_UPDATE_CALLS);
3420
3421                 mutex_unlock(&ftrace_lock);
3422         }
3423         free_ftrace_hash(iter->hash);
3424         kfree(iter);
3425
3426         mutex_unlock(&ftrace_regex_lock);
3427         return 0;
3428 }
3429
3430 static const struct file_operations ftrace_avail_fops = {
3431         .open = ftrace_avail_open,
3432         .read = seq_read,
3433         .llseek = seq_lseek,
3434         .release = seq_release_private,
3435 };
3436
3437 static const struct file_operations ftrace_enabled_fops = {
3438         .open = ftrace_enabled_open,
3439         .read = seq_read,
3440         .llseek = seq_lseek,
3441         .release = seq_release_private,
3442 };
3443
3444 static const struct file_operations ftrace_filter_fops = {
3445         .open = ftrace_filter_open,
3446         .read = seq_read,
3447         .write = ftrace_filter_write,
3448         .llseek = ftrace_regex_lseek,
3449         .release = ftrace_regex_release,
3450 };
3451
3452 static const struct file_operations ftrace_notrace_fops = {
3453         .open = ftrace_notrace_open,
3454         .read = seq_read,
3455         .write = ftrace_notrace_write,
3456         .llseek = ftrace_regex_lseek,
3457         .release = ftrace_regex_release,
3458 };
3459
3460 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3461
3462 static DEFINE_MUTEX(graph_lock);
3463
3464 int ftrace_graph_count;
3465 int ftrace_graph_filter_enabled;
3466 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3467
3468 static void *
3469 __g_next(struct seq_file *m, loff_t *pos)
3470 {
3471         if (*pos >= ftrace_graph_count)
3472                 return NULL;
3473         return &ftrace_graph_funcs[*pos];
3474 }
3475
3476 static void *
3477 g_next(struct seq_file *m, void *v, loff_t *pos)
3478 {
3479         (*pos)++;
3480         return __g_next(m, pos);
3481 }
3482
3483 static void *g_start(struct seq_file *m, loff_t *pos)
3484 {
3485         mutex_lock(&graph_lock);
3486
3487         /* Nothing, tell g_show to print all functions are enabled */
3488         if (!ftrace_graph_filter_enabled && !*pos)
3489                 return (void *)1;
3490
3491         return __g_next(m, pos);
3492 }
3493
3494 static void g_stop(struct seq_file *m, void *p)
3495 {
3496         mutex_unlock(&graph_lock);
3497 }
3498
3499 static int g_show(struct seq_file *m, void *v)
3500 {
3501         unsigned long *ptr = v;
3502
3503         if (!ptr)
3504                 return 0;
3505
3506         if (ptr == (unsigned long *)1) {
3507                 seq_printf(m, "#### all functions enabled ####\n");
3508                 return 0;
3509         }
3510
3511         seq_printf(m, "%ps\n", (void *)*ptr);
3512
3513         return 0;
3514 }
3515
3516 static const struct seq_operations ftrace_graph_seq_ops = {
3517         .start = g_start,
3518         .next = g_next,
3519         .stop = g_stop,
3520         .show = g_show,
3521 };
3522
3523 static int
3524 ftrace_graph_open(struct inode *inode, struct file *file)
3525 {
3526         int ret = 0;
3527
3528         if (unlikely(ftrace_disabled))
3529                 return -ENODEV;
3530
3531         mutex_lock(&graph_lock);
3532         if ((file->f_mode & FMODE_WRITE) &&
3533             (file->f_flags & O_TRUNC)) {
3534                 ftrace_graph_filter_enabled = 0;
3535                 ftrace_graph_count = 0;
3536                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
3537         }
3538         mutex_unlock(&graph_lock);
3539
3540         if (file->f_mode & FMODE_READ)
3541                 ret = seq_open(file, &ftrace_graph_seq_ops);
3542
3543         return ret;
3544 }
3545
3546 static int
3547 ftrace_graph_release(struct inode *inode, struct file *file)
3548 {
3549         if (file->f_mode & FMODE_READ)
3550                 seq_release(inode, file);
3551         return 0;
3552 }
3553
3554 static int
3555 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3556 {
3557         struct dyn_ftrace *rec;
3558         struct ftrace_page *pg;
3559         int search_len;
3560         int fail = 1;
3561         int type, not;
3562         char *search;
3563         bool exists;
3564         int i;
3565
3566         /* decode regex */
3567         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3568         if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
3569                 return -EBUSY;
3570
3571         search_len = strlen(search);
3572
3573         mutex_lock(&ftrace_lock);
3574
3575         if (unlikely(ftrace_disabled)) {
3576                 mutex_unlock(&ftrace_lock);
3577                 return -ENODEV;
3578         }
3579
3580         do_for_each_ftrace_rec(pg, rec) {
3581
3582                 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3583                         /* if it is in the array */
3584                         exists = false;
3585                         for (i = 0; i < *idx; i++) {
3586                                 if (array[i] == rec->ip) {
3587                                         exists = true;
3588                                         break;
3589                                 }
3590                         }
3591
3592                         if (!not) {
3593                                 fail = 0;
3594                                 if (!exists) {
3595                                         array[(*idx)++] = rec->ip;
3596                                         if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
3597                                                 goto out;
3598                                 }
3599                         } else {
3600                                 if (exists) {
3601                                         array[i] = array[--(*idx)];
3602                                         array[*idx] = 0;
3603                                         fail = 0;
3604                                 }
3605                         }
3606                 }
3607         } while_for_each_ftrace_rec();
3608 out:
3609         mutex_unlock(&ftrace_lock);
3610
3611         if (fail)
3612                 return -EINVAL;
3613
3614         ftrace_graph_filter_enabled = 1;
3615         return 0;
3616 }
3617
3618 static ssize_t
3619 ftrace_graph_write(struct file *file, const char __user *ubuf,
3620                    size_t cnt, loff_t *ppos)
3621 {
3622         struct trace_parser parser;
3623         ssize_t read, ret;
3624
3625         if (!cnt)
3626                 return 0;
3627
3628         mutex_lock(&graph_lock);
3629
3630         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
3631                 ret = -ENOMEM;
3632                 goto out_unlock;
3633         }
3634
3635         read = trace_get_user(&parser, ubuf, cnt, ppos);
3636
3637         if (read >= 0 && trace_parser_loaded((&parser))) {
3638                 parser.buffer[parser.idx] = 0;
3639
3640                 /* we allow only one expression at a time */
3641                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3642                                         parser.buffer);
3643                 if (ret)
3644                         goto out_free;
3645         }
3646
3647         ret = read;
3648
3649 out_free:
3650         trace_parser_put(&parser);
3651 out_unlock:
3652         mutex_unlock(&graph_lock);
3653
3654         return ret;
3655 }
3656
3657 static const struct file_operations ftrace_graph_fops = {
3658         .open           = ftrace_graph_open,
3659         .read           = seq_read,
3660         .write          = ftrace_graph_write,
3661         .release        = ftrace_graph_release,
3662         .llseek         = seq_lseek,
3663 };
3664 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3665
3666 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3667 {
3668
3669         trace_create_file("available_filter_functions", 0444,
3670                         d_tracer, NULL, &ftrace_avail_fops);
3671
3672         trace_create_file("enabled_functions", 0444,
3673                         d_tracer, NULL, &ftrace_enabled_fops);
3674
3675         trace_create_file("set_ftrace_filter", 0644, d_tracer,
3676                         NULL, &ftrace_filter_fops);
3677
3678         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3679                                     NULL, &ftrace_notrace_fops);
3680
3681 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3682         trace_create_file("set_graph_function", 0444, d_tracer,
3683                                     NULL,
3684                                     &ftrace_graph_fops);
3685 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3686
3687         return 0;
3688 }
3689
3690 static void ftrace_swap_recs(void *a, void *b, int size)
3691 {
3692         struct dyn_ftrace *reca = a;
3693         struct dyn_ftrace *recb = b;
3694         struct dyn_ftrace t;
3695
3696         t = *reca;
3697         *reca = *recb;
3698         *recb = t;
3699 }
3700
3701 static int ftrace_process_locs(struct module *mod,
3702                                unsigned long *start,
3703                                unsigned long *end)
3704 {
3705         struct ftrace_page *pg;
3706         unsigned long count;
3707         unsigned long *p;
3708         unsigned long addr;
3709         unsigned long flags = 0; /* Shut up gcc */
3710         int ret = -ENOMEM;
3711
3712         count = end - start;
3713
3714         if (!count)
3715                 return 0;
3716
3717         pg = ftrace_allocate_pages(count);
3718         if (!pg)
3719                 return -ENOMEM;
3720
3721         mutex_lock(&ftrace_lock);
3722
3723         /*
3724          * Core and each module needs their own pages, as
3725          * modules will free them when they are removed.
3726          * Force a new page to be allocated for modules.
3727          */
3728         if (!mod) {
3729                 WARN_ON(ftrace_pages || ftrace_pages_start);
3730                 /* First initialization */
3731                 ftrace_pages = ftrace_pages_start = pg;
3732         } else {
3733                 if (!ftrace_pages)
3734                         goto out;
3735
3736                 if (WARN_ON(ftrace_pages->next)) {
3737                         /* Hmm, we have free pages? */
3738                         while (ftrace_pages->next)
3739                                 ftrace_pages = ftrace_pages->next;
3740                 }
3741
3742                 ftrace_pages->next = pg;
3743                 ftrace_pages = pg;
3744         }
3745
3746         p = start;
3747         while (p < end) {
3748                 addr = ftrace_call_adjust(*p++);
3749                 /*
3750                  * Some architecture linkers will pad between
3751                  * the different mcount_loc sections of different
3752                  * object files to satisfy alignments.
3753                  * Skip any NULL pointers.
3754                  */
3755                 if (!addr)
3756                         continue;
3757                 if (!ftrace_record_ip(addr))
3758                         break;
3759         }
3760
3761         /* These new locations need to be initialized */
3762         ftrace_new_pgs = pg;
3763
3764         /* Make each individual set of pages sorted by ips */
3765         for (; pg; pg = pg->next)
3766                 sort(pg->records, pg->index, sizeof(struct dyn_ftrace),
3767                      ftrace_cmp_recs, ftrace_swap_recs);
3768
3769         /*
3770          * We only need to disable interrupts on start up
3771          * because we are modifying code that an interrupt
3772          * may execute, and the modification is not atomic.
3773          * But for modules, nothing runs the code we modify
3774          * until we are finished with it, and there's no
3775          * reason to cause large interrupt latencies while we do it.
3776          */
3777         if (!mod)
3778                 local_irq_save(flags);
3779         ftrace_update_code(mod);
3780         if (!mod)
3781                 local_irq_restore(flags);
3782         ret = 0;
3783  out:
3784         mutex_unlock(&ftrace_lock);
3785
3786         return ret;
3787 }
3788
3789 #ifdef CONFIG_MODULES
3790
3791 #define next_to_ftrace_page(p) container_of(p, struct ftrace_page, next)
3792
3793 void ftrace_release_mod(struct module *mod)
3794 {
3795         struct dyn_ftrace *rec;
3796         struct ftrace_page **last_pg;
3797         struct ftrace_page *pg;
3798         int order;
3799
3800         mutex_lock(&ftrace_lock);
3801
3802         if (ftrace_disabled)
3803                 goto out_unlock;
3804
3805         /*
3806          * Each module has its own ftrace_pages, remove
3807          * them from the list.
3808          */
3809         last_pg = &ftrace_pages_start;
3810         for (pg = ftrace_pages_start; pg; pg = *last_pg) {
3811                 rec = &pg->records[0];
3812                 if (within_module_core(rec->ip, mod)) {
3813                         /*
3814                          * As core pages are first, the first
3815                          * page should never be a module page.
3816                          */
3817                         if (WARN_ON(pg == ftrace_pages_start))
3818                                 goto out_unlock;
3819
3820                         /* Check if we are deleting the last page */
3821                         if (pg == ftrace_pages)
3822                                 ftrace_pages = next_to_ftrace_page(last_pg);
3823
3824                         *last_pg = pg->next;
3825                         order = get_count_order(pg->size / ENTRIES_PER_PAGE);
3826                         free_pages((unsigned long)pg->records, order);
3827                         kfree(pg);
3828                 } else
3829                         last_pg = &pg->next;
3830         }
3831  out_unlock:
3832         mutex_unlock(&ftrace_lock);
3833 }
3834
3835 static void ftrace_init_module(struct module *mod,
3836                                unsigned long *start, unsigned long *end)
3837 {
3838         if (ftrace_disabled || start == end)
3839                 return;
3840         ftrace_process_locs(mod, start, end);
3841 }
3842
3843 static int ftrace_module_notify(struct notifier_block *self,
3844                                 unsigned long val, void *data)
3845 {
3846         struct module *mod = data;
3847
3848         switch (val) {
3849         case MODULE_STATE_COMING:
3850                 ftrace_init_module(mod, mod->ftrace_callsites,
3851                                    mod->ftrace_callsites +
3852                                    mod->num_ftrace_callsites);
3853                 break;
3854         case MODULE_STATE_GOING:
3855                 ftrace_release_mod(mod);
3856                 break;
3857         }
3858
3859         return 0;
3860 }
3861 #else
3862 static int ftrace_module_notify(struct notifier_block *self,
3863                                 unsigned long val, void *data)
3864 {
3865         return 0;
3866 }
3867 #endif /* CONFIG_MODULES */
3868
3869 struct notifier_block ftrace_module_nb = {
3870         .notifier_call = ftrace_module_notify,
3871         .priority = 0,
3872 };
3873
3874 extern unsigned long __start_mcount_loc[];
3875 extern unsigned long __stop_mcount_loc[];
3876
3877 void __init ftrace_init(void)
3878 {
3879         unsigned long count, addr, flags;
3880         int ret;
3881
3882         /* Keep the ftrace pointer to the stub */
3883         addr = (unsigned long)ftrace_stub;
3884
3885         local_irq_save(flags);
3886         ftrace_dyn_arch_init(&addr);
3887         local_irq_restore(flags);
3888
3889         /* ftrace_dyn_arch_init places the return code in addr */
3890         if (addr)
3891                 goto failed;
3892
3893         count = __stop_mcount_loc - __start_mcount_loc;
3894
3895         ret = ftrace_dyn_table_alloc(count);
3896         if (ret)
3897                 goto failed;
3898
3899         last_ftrace_enabled = ftrace_enabled = 1;
3900
3901         ret = ftrace_process_locs(NULL,
3902                                   __start_mcount_loc,
3903                                   __stop_mcount_loc);
3904
3905         ret = register_module_notifier(&ftrace_module_nb);
3906         if (ret)
3907                 pr_warning("Failed to register trace ftrace module notifier\n");
3908
3909         set_ftrace_early_filters();
3910
3911         return;
3912  failed:
3913         ftrace_disabled = 1;
3914 }
3915
3916 #else
3917
3918 static struct ftrace_ops global_ops = {
3919         .func                   = ftrace_stub,
3920 };
3921
3922 static int __init ftrace_nodyn_init(void)
3923 {
3924         ftrace_enabled = 1;
3925         return 0;
3926 }
3927 device_initcall(ftrace_nodyn_init);
3928
3929 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
3930 static inline void ftrace_startup_enable(int command) { }
3931 /* Keep as macros so we do not need to define the commands */
3932 # define ftrace_startup(ops, command)                   \
3933         ({                                              \
3934                 (ops)->flags |= FTRACE_OPS_FL_ENABLED;  \
3935                 0;                                      \
3936         })
3937 # define ftrace_shutdown(ops, command)  do { } while (0)
3938 # define ftrace_startup_sysctl()        do { } while (0)
3939 # define ftrace_shutdown_sysctl()       do { } while (0)
3940
3941 static inline int
3942 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
3943 {
3944         return 1;
3945 }
3946
3947 #endif /* CONFIG_DYNAMIC_FTRACE */
3948
3949 static void
3950 ftrace_ops_control_func(unsigned long ip, unsigned long parent_ip)
3951 {
3952         struct ftrace_ops *op;
3953
3954         if (unlikely(trace_recursion_test(TRACE_CONTROL_BIT)))
3955                 return;
3956
3957         /*
3958          * Some of the ops may be dynamically allocated,
3959          * they must be freed after a synchronize_sched().
3960          */
3961         preempt_disable_notrace();
3962         trace_recursion_set(TRACE_CONTROL_BIT);
3963         op = rcu_dereference_raw(ftrace_control_list);
3964         while (op != &ftrace_list_end) {
3965                 if (!ftrace_function_local_disabled(op) &&
3966                     ftrace_ops_test(op, ip))
3967                         op->func(ip, parent_ip);
3968
3969                 op = rcu_dereference_raw(op->next);
3970         };
3971         trace_recursion_clear(TRACE_CONTROL_BIT);
3972         preempt_enable_notrace();
3973 }
3974
3975 static struct ftrace_ops control_ops = {
3976         .func = ftrace_ops_control_func,
3977 };
3978
3979 static void
3980 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
3981 {
3982         struct ftrace_ops *op;
3983
3984         if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
3985                 return;
3986
3987         trace_recursion_set(TRACE_INTERNAL_BIT);
3988         /*
3989          * Some of the ops may be dynamically allocated,
3990          * they must be freed after a synchronize_sched().
3991          */
3992         preempt_disable_notrace();
3993         op = rcu_dereference_raw(ftrace_ops_list);
3994         while (op != &ftrace_list_end) {
3995                 if (ftrace_ops_test(op, ip))
3996                         op->func(ip, parent_ip);
3997                 op = rcu_dereference_raw(op->next);
3998         };
3999         preempt_enable_notrace();
4000         trace_recursion_clear(TRACE_INTERNAL_BIT);
4001 }
4002
4003 static void clear_ftrace_swapper(void)
4004 {
4005         struct task_struct *p;
4006         int cpu;
4007
4008         get_online_cpus();
4009         for_each_online_cpu(cpu) {
4010                 p = idle_task(cpu);
4011                 clear_tsk_trace_trace(p);
4012         }
4013         put_online_cpus();
4014 }
4015
4016 static void set_ftrace_swapper(void)
4017 {
4018         struct task_struct *p;
4019         int cpu;
4020
4021         get_online_cpus();
4022         for_each_online_cpu(cpu) {
4023                 p = idle_task(cpu);
4024                 set_tsk_trace_trace(p);
4025         }
4026         put_online_cpus();
4027 }
4028
4029 static void clear_ftrace_pid(struct pid *pid)
4030 {
4031         struct task_struct *p;
4032
4033         rcu_read_lock();
4034         do_each_pid_task(pid, PIDTYPE_PID, p) {
4035                 clear_tsk_trace_trace(p);
4036         } while_each_pid_task(pid, PIDTYPE_PID, p);
4037         rcu_read_unlock();
4038
4039         put_pid(pid);
4040 }
4041
4042 static void set_ftrace_pid(struct pid *pid)
4043 {
4044         struct task_struct *p;
4045
4046         rcu_read_lock();
4047         do_each_pid_task(pid, PIDTYPE_PID, p) {
4048                 set_tsk_trace_trace(p);
4049         } while_each_pid_task(pid, PIDTYPE_PID, p);
4050         rcu_read_unlock();
4051 }
4052
4053 static void clear_ftrace_pid_task(struct pid *pid)
4054 {
4055         if (pid == ftrace_swapper_pid)
4056                 clear_ftrace_swapper();
4057         else
4058                 clear_ftrace_pid(pid);
4059 }
4060
4061 static void set_ftrace_pid_task(struct pid *pid)
4062 {
4063         if (pid == ftrace_swapper_pid)
4064                 set_ftrace_swapper();
4065         else
4066                 set_ftrace_pid(pid);
4067 }
4068
4069 static int ftrace_pid_add(int p)
4070 {
4071         struct pid *pid;
4072         struct ftrace_pid *fpid;
4073         int ret = -EINVAL;
4074
4075         mutex_lock(&ftrace_lock);
4076
4077         if (!p)
4078                 pid = ftrace_swapper_pid;
4079         else
4080                 pid = find_get_pid(p);
4081
4082         if (!pid)
4083                 goto out;
4084
4085         ret = 0;
4086
4087         list_for_each_entry(fpid, &ftrace_pids, list)
4088                 if (fpid->pid == pid)
4089                         goto out_put;
4090
4091         ret = -ENOMEM;
4092
4093         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
4094         if (!fpid)
4095                 goto out_put;
4096
4097         list_add(&fpid->list, &ftrace_pids);
4098         fpid->pid = pid;
4099
4100         set_ftrace_pid_task(pid);
4101
4102         ftrace_update_pid_func();
4103         ftrace_startup_enable(0);
4104
4105         mutex_unlock(&ftrace_lock);
4106         return 0;
4107
4108 out_put:
4109         if (pid != ftrace_swapper_pid)
4110                 put_pid(pid);
4111
4112 out:
4113         mutex_unlock(&ftrace_lock);
4114         return ret;
4115 }
4116
4117 static void ftrace_pid_reset(void)
4118 {
4119         struct ftrace_pid *fpid, *safe;
4120
4121         mutex_lock(&ftrace_lock);
4122         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
4123                 struct pid *pid = fpid->pid;
4124
4125                 clear_ftrace_pid_task(pid);
4126
4127                 list_del(&fpid->list);
4128                 kfree(fpid);
4129         }
4130
4131         ftrace_update_pid_func();
4132         ftrace_startup_enable(0);
4133
4134         mutex_unlock(&ftrace_lock);
4135 }
4136
4137 static void *fpid_start(struct seq_file *m, loff_t *pos)
4138 {
4139         mutex_lock(&ftrace_lock);
4140
4141         if (list_empty(&ftrace_pids) && (!*pos))
4142                 return (void *) 1;
4143
4144         return seq_list_start(&ftrace_pids, *pos);
4145 }
4146
4147 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
4148 {
4149         if (v == (void *)1)
4150                 return NULL;
4151
4152         return seq_list_next(v, &ftrace_pids, pos);
4153 }
4154
4155 static void fpid_stop(struct seq_file *m, void *p)
4156 {
4157         mutex_unlock(&ftrace_lock);
4158 }
4159
4160 static int fpid_show(struct seq_file *m, void *v)
4161 {
4162         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
4163
4164         if (v == (void *)1) {
4165                 seq_printf(m, "no pid\n");
4166                 return 0;
4167         }
4168
4169         if (fpid->pid == ftrace_swapper_pid)
4170                 seq_printf(m, "swapper tasks\n");
4171         else
4172                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
4173
4174         return 0;
4175 }
4176
4177 static const struct seq_operations ftrace_pid_sops = {
4178         .start = fpid_start,
4179         .next = fpid_next,
4180         .stop = fpid_stop,
4181         .show = fpid_show,
4182 };
4183
4184 static int
4185 ftrace_pid_open(struct inode *inode, struct file *file)
4186 {
4187         int ret = 0;
4188
4189         if ((file->f_mode & FMODE_WRITE) &&
4190             (file->f_flags & O_TRUNC))
4191                 ftrace_pid_reset();
4192
4193         if (file->f_mode & FMODE_READ)
4194                 ret = seq_open(file, &ftrace_pid_sops);
4195
4196         return ret;
4197 }
4198
4199 static ssize_t
4200 ftrace_pid_write(struct file *filp, const char __user *ubuf,
4201                    size_t cnt, loff_t *ppos)
4202 {
4203         char buf[64], *tmp;
4204         long val;
4205         int ret;
4206
4207         if (cnt >= sizeof(buf))
4208                 return -EINVAL;
4209
4210         if (copy_from_user(&buf, ubuf, cnt))
4211                 return -EFAULT;
4212
4213         buf[cnt] = 0;
4214
4215         /*
4216          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
4217          * to clean the filter quietly.
4218          */
4219         tmp = strstrip(buf);
4220         if (strlen(tmp) == 0)
4221                 return 1;
4222
4223         ret = strict_strtol(tmp, 10, &val);
4224         if (ret < 0)
4225                 return ret;
4226
4227         ret = ftrace_pid_add(val);
4228
4229         return ret ? ret : cnt;
4230 }
4231
4232 static int
4233 ftrace_pid_release(struct inode *inode, struct file *file)
4234 {
4235         if (file->f_mode & FMODE_READ)
4236                 seq_release(inode, file);
4237
4238         return 0;
4239 }
4240
4241 static const struct file_operations ftrace_pid_fops = {
4242         .open           = ftrace_pid_open,
4243         .write          = ftrace_pid_write,
4244         .read           = seq_read,
4245         .llseek         = seq_lseek,
4246         .release        = ftrace_pid_release,
4247 };
4248
4249 static __init int ftrace_init_debugfs(void)
4250 {
4251         struct dentry *d_tracer;
4252
4253         d_tracer = tracing_init_dentry();
4254         if (!d_tracer)
4255                 return 0;
4256
4257         ftrace_init_dyn_debugfs(d_tracer);
4258
4259         trace_create_file("set_ftrace_pid", 0644, d_tracer,
4260                             NULL, &ftrace_pid_fops);
4261
4262         ftrace_profile_debugfs(d_tracer);
4263
4264         return 0;
4265 }
4266 fs_initcall(ftrace_init_debugfs);
4267
4268 /**
4269  * ftrace_kill - kill ftrace
4270  *
4271  * This function should be used by panic code. It stops ftrace
4272  * but in a not so nice way. If you need to simply kill ftrace
4273  * from a non-atomic section, use ftrace_kill.
4274  */
4275 void ftrace_kill(void)
4276 {
4277         ftrace_disabled = 1;
4278         ftrace_enabled = 0;
4279         clear_ftrace_function();
4280 }
4281
4282 /**
4283  * Test if ftrace is dead or not.
4284  */
4285 int ftrace_is_dead(void)
4286 {
4287         return ftrace_disabled;
4288 }
4289
4290 /**
4291  * register_ftrace_function - register a function for profiling
4292  * @ops - ops structure that holds the function for profiling.
4293  *
4294  * Register a function to be called by all functions in the
4295  * kernel.
4296  *
4297  * Note: @ops->func and all the functions it calls must be labeled
4298  *       with "notrace", otherwise it will go into a
4299  *       recursive loop.
4300  */
4301 int register_ftrace_function(struct ftrace_ops *ops)
4302 {
4303         int ret = -1;
4304
4305         mutex_lock(&ftrace_lock);
4306
4307         if (unlikely(ftrace_disabled))
4308                 goto out_unlock;
4309
4310         ret = __register_ftrace_function(ops);
4311         if (!ret)
4312                 ret = ftrace_startup(ops, 0);
4313
4314
4315  out_unlock:
4316         mutex_unlock(&ftrace_lock);
4317         return ret;
4318 }
4319 EXPORT_SYMBOL_GPL(register_ftrace_function);
4320
4321 /**
4322  * unregister_ftrace_function - unregister a function for profiling.
4323  * @ops - ops structure that holds the function to unregister
4324  *
4325  * Unregister a function that was added to be called by ftrace profiling.
4326  */
4327 int unregister_ftrace_function(struct ftrace_ops *ops)
4328 {
4329         int ret;
4330
4331         mutex_lock(&ftrace_lock);
4332         ret = __unregister_ftrace_function(ops);
4333         if (!ret)
4334                 ftrace_shutdown(ops, 0);
4335         mutex_unlock(&ftrace_lock);
4336
4337         return ret;
4338 }
4339 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
4340
4341 int
4342 ftrace_enable_sysctl(struct ctl_table *table, int write,
4343                      void __user *buffer, size_t *lenp,
4344                      loff_t *ppos)
4345 {
4346         int ret = -ENODEV;
4347
4348         mutex_lock(&ftrace_lock);
4349
4350         if (unlikely(ftrace_disabled))
4351                 goto out;
4352
4353         ret = proc_dointvec(table, write, buffer, lenp, ppos);
4354
4355         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
4356                 goto out;
4357
4358         last_ftrace_enabled = !!ftrace_enabled;
4359
4360         if (ftrace_enabled) {
4361
4362                 ftrace_startup_sysctl();
4363
4364                 /* we are starting ftrace again */
4365                 if (ftrace_ops_list != &ftrace_list_end) {
4366                         if (ftrace_ops_list->next == &ftrace_list_end)
4367                                 ftrace_trace_function = ftrace_ops_list->func;
4368                         else
4369                                 ftrace_trace_function = ftrace_ops_list_func;
4370                 }
4371
4372         } else {
4373                 /* stopping ftrace calls (just send to ftrace_stub) */
4374                 ftrace_trace_function = ftrace_stub;
4375
4376                 ftrace_shutdown_sysctl();
4377         }
4378
4379  out:
4380         mutex_unlock(&ftrace_lock);
4381         return ret;
4382 }
4383
4384 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4385
4386 static int ftrace_graph_active;
4387 static struct notifier_block ftrace_suspend_notifier;
4388
4389 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
4390 {
4391         return 0;
4392 }
4393
4394 /* The callbacks that hook a function */
4395 trace_func_graph_ret_t ftrace_graph_return =
4396                         (trace_func_graph_ret_t)ftrace_stub;
4397 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
4398
4399 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
4400 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
4401 {
4402         int i;
4403         int ret = 0;
4404         unsigned long flags;
4405         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
4406         struct task_struct *g, *t;
4407
4408         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
4409                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
4410                                         * sizeof(struct ftrace_ret_stack),
4411                                         GFP_KERNEL);
4412                 if (!ret_stack_list[i]) {
4413                         start = 0;
4414                         end = i;
4415                         ret = -ENOMEM;
4416                         goto free;
4417                 }
4418         }
4419
4420         read_lock_irqsave(&tasklist_lock, flags);
4421         do_each_thread(g, t) {
4422                 if (start == end) {
4423                         ret = -EAGAIN;
4424                         goto unlock;
4425                 }
4426
4427                 if (t->ret_stack == NULL) {
4428                         atomic_set(&t->tracing_graph_pause, 0);
4429                         atomic_set(&t->trace_overrun, 0);
4430                         t->curr_ret_stack = -1;
4431                         /* Make sure the tasks see the -1 first: */
4432                         smp_wmb();
4433                         t->ret_stack = ret_stack_list[start++];
4434                 }
4435         } while_each_thread(g, t);
4436
4437 unlock:
4438         read_unlock_irqrestore(&tasklist_lock, flags);
4439 free:
4440         for (i = start; i < end; i++)
4441                 kfree(ret_stack_list[i]);
4442         return ret;
4443 }
4444
4445 static void
4446 ftrace_graph_probe_sched_switch(void *ignore,
4447                         struct task_struct *prev, struct task_struct *next)
4448 {
4449         unsigned long long timestamp;
4450         int index;
4451
4452         /*
4453          * Does the user want to count the time a function was asleep.
4454          * If so, do not update the time stamps.
4455          */
4456         if (trace_flags & TRACE_ITER_SLEEP_TIME)
4457                 return;
4458
4459         timestamp = trace_clock_local();
4460
4461         prev->ftrace_timestamp = timestamp;
4462
4463         /* only process tasks that we timestamped */
4464         if (!next->ftrace_timestamp)
4465                 return;
4466
4467         /*
4468          * Update all the counters in next to make up for the
4469          * time next was sleeping.
4470          */
4471         timestamp -= next->ftrace_timestamp;
4472
4473         for (index = next->curr_ret_stack; index >= 0; index--)
4474                 next->ret_stack[index].calltime += timestamp;
4475 }
4476
4477 /* Allocate a return stack for each task */
4478 static int start_graph_tracing(void)
4479 {
4480         struct ftrace_ret_stack **ret_stack_list;
4481         int ret, cpu;
4482
4483         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
4484                                 sizeof(struct ftrace_ret_stack *),
4485                                 GFP_KERNEL);
4486
4487         if (!ret_stack_list)
4488                 return -ENOMEM;
4489
4490         /* The cpu_boot init_task->ret_stack will never be freed */
4491         for_each_online_cpu(cpu) {
4492                 if (!idle_task(cpu)->ret_stack)
4493                         ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4494         }
4495
4496         do {
4497                 ret = alloc_retstack_tasklist(ret_stack_list);
4498         } while (ret == -EAGAIN);
4499
4500         if (!ret) {
4501                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4502                 if (ret)
4503                         pr_info("ftrace_graph: Couldn't activate tracepoint"
4504                                 " probe to kernel_sched_switch\n");
4505         }
4506
4507         kfree(ret_stack_list);
4508         return ret;
4509 }
4510
4511 /*
4512  * Hibernation protection.
4513  * The state of the current task is too much unstable during
4514  * suspend/restore to disk. We want to protect against that.
4515  */
4516 static int
4517 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
4518                                                         void *unused)
4519 {
4520         switch (state) {
4521         case PM_HIBERNATION_PREPARE:
4522                 pause_graph_tracing();
4523                 break;
4524
4525         case PM_POST_HIBERNATION:
4526                 unpause_graph_tracing();
4527                 break;
4528         }
4529         return NOTIFY_DONE;
4530 }
4531
4532 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
4533                         trace_func_graph_ent_t entryfunc)
4534 {
4535         int ret = 0;
4536
4537         mutex_lock(&ftrace_lock);
4538
4539         /* we currently allow only one tracer registered at a time */
4540         if (ftrace_graph_active) {
4541                 ret = -EBUSY;
4542                 goto out;
4543         }
4544
4545         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
4546         register_pm_notifier(&ftrace_suspend_notifier);
4547
4548         ftrace_graph_active++;
4549         ret = start_graph_tracing();
4550         if (ret) {
4551                 ftrace_graph_active--;
4552                 goto out;
4553         }
4554
4555         ftrace_graph_return = retfunc;
4556         ftrace_graph_entry = entryfunc;
4557
4558         ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
4559
4560 out:
4561         mutex_unlock(&ftrace_lock);
4562         return ret;
4563 }
4564
4565 void unregister_ftrace_graph(void)
4566 {
4567         mutex_lock(&ftrace_lock);
4568
4569         if (unlikely(!ftrace_graph_active))
4570                 goto out;
4571
4572         ftrace_graph_active--;
4573         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4574         ftrace_graph_entry = ftrace_graph_entry_stub;
4575         ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
4576         unregister_pm_notifier(&ftrace_suspend_notifier);
4577         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4578
4579  out:
4580         mutex_unlock(&ftrace_lock);
4581 }
4582
4583 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
4584
4585 static void
4586 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
4587 {
4588         atomic_set(&t->tracing_graph_pause, 0);
4589         atomic_set(&t->trace_overrun, 0);
4590         t->ftrace_timestamp = 0;
4591         /* make curr_ret_stack visible before we add the ret_stack */
4592         smp_wmb();
4593         t->ret_stack = ret_stack;
4594 }
4595
4596 /*
4597  * Allocate a return stack for the idle task. May be the first
4598  * time through, or it may be done by CPU hotplug online.
4599  */
4600 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
4601 {
4602         t->curr_ret_stack = -1;
4603         /*
4604          * The idle task has no parent, it either has its own
4605          * stack or no stack at all.
4606          */
4607         if (t->ret_stack)
4608                 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
4609
4610         if (ftrace_graph_active) {
4611                 struct ftrace_ret_stack *ret_stack;
4612
4613                 ret_stack = per_cpu(idle_ret_stack, cpu);
4614                 if (!ret_stack) {
4615                         ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4616                                             * sizeof(struct ftrace_ret_stack),
4617                                             GFP_KERNEL);
4618                         if (!ret_stack)
4619                                 return;
4620                         per_cpu(idle_ret_stack, cpu) = ret_stack;
4621                 }
4622                 graph_init_task(t, ret_stack);
4623         }
4624 }
4625
4626 /* Allocate a return stack for newly created task */
4627 void ftrace_graph_init_task(struct task_struct *t)
4628 {
4629         /* Make sure we do not use the parent ret_stack */
4630         t->ret_stack = NULL;
4631         t->curr_ret_stack = -1;
4632
4633         if (ftrace_graph_active) {
4634                 struct ftrace_ret_stack *ret_stack;
4635
4636                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4637                                 * sizeof(struct ftrace_ret_stack),
4638                                 GFP_KERNEL);
4639                 if (!ret_stack)
4640                         return;
4641                 graph_init_task(t, ret_stack);
4642         }
4643 }
4644
4645 void ftrace_graph_exit_task(struct task_struct *t)
4646 {
4647         struct ftrace_ret_stack *ret_stack = t->ret_stack;
4648
4649         t->ret_stack = NULL;
4650         /* NULL must become visible to IRQs before we free it: */
4651         barrier();
4652
4653         kfree(ret_stack);
4654 }
4655
4656 void ftrace_graph_stop(void)
4657 {
4658         ftrace_stop();
4659 }
4660 #endif