kernel/trace/trace_functions.c

   1 /*
   2  * ring buffer based function tracer
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Based on code from the latency_tracer, that is:
   8  *
   9  *  Copyright (C) 2004-2006 Ingo Molnar
  10  *  Copyright (C) 2004 Nadia Yvette Chambers
  11  */
  12 #include <linux/ring_buffer.h>
  13 #include <linux/debugfs.h>
  14 #include <linux/uaccess.h>
  15 #include <linux/ftrace.h>
  16 #include <linux/slab.h>
  17 #include <linux/fs.h>
  18
  19 #include "trace.h"
  20
  21 static void tracing_start_function_trace(struct trace_array *tr);
  22 static void tracing_stop_function_trace(struct trace_array *tr);
  23 static void
  24 function_trace_call(unsigned long ip, unsigned long parent_ip,
  25                     struct ftrace_ops *op, struct pt_regs *pt_regs);
  26 static void
  27 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
  28                           struct ftrace_ops *op, struct pt_regs *pt_regs);
  29 static struct ftrace_ops trace_ops;
  30 static struct ftrace_ops trace_stack_ops;
  31 static struct tracer_flags func_flags;
  32
  33 /* Our option */
  34 enum {
  35         TRACE_FUNC_OPT_STACK    = 0x1,
  36 };
  37
  38 static int allocate_ftrace_ops(struct trace_array *tr)
  39 {
  40         struct ftrace_ops *ops;
  41
  42         ops = kzalloc(sizeof(*ops), GFP_KERNEL);
  43         if (!ops)
  44                 return -ENOMEM;
  45
  46         /* Currently only the non stack verision is supported */
  47         ops->func = function_trace_call;
  48         ops->flags = FTRACE_OPS_FL_RECURSION_SAFE;
  49
  50         tr->ops = ops;
  51         ops->private = tr;
  52         return 0;
  53 }
  54
  55
  56 int ftrace_create_function_files(struct trace_array *tr,
  57                                  struct dentry *parent)
  58 {
  59         int ret;
  60
  61         /*
  62          * The top level array uses the "global_ops", and the files are
  63          * created on boot up.
  64          */
  65         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
  66                 return 0;
  67
  68         ret = allocate_ftrace_ops(tr);
  69         if (ret)
  70                 return ret;
  71
  72         ftrace_create_filter_files(tr->ops, parent);
  73
  74         return 0;
  75 }
  76
  77 void ftrace_destroy_function_files(struct trace_array *tr)
  78 {
  79         ftrace_destroy_filter_files(tr->ops);
  80         kfree(tr->ops);
  81         tr->ops = NULL;
  82 }
  83
  84 static int function_trace_init(struct trace_array *tr)
  85 {
  86         struct ftrace_ops *ops;
  87
  88         if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
  89                 /* There's only one global tr */
  90                 if (!trace_ops.private) {
  91                         trace_ops.private = tr;
  92                         trace_stack_ops.private = tr;
  93                 }
  94
  95                 if (func_flags.val & TRACE_FUNC_OPT_STACK)
  96                         ops = &trace_stack_ops;
  97                 else
  98                         ops = &trace_ops;
  99                 tr->ops = ops;
 100         } else if (!tr->ops) {
 101                 /*
 102                  * Instance trace_arrays get their ops allocated
 103                  * at instance creation. Unless it failed
 104                  * the allocation.
 105                  */
 106                 return -ENOMEM;
 107         }
 108
 109         tr->trace_buffer.cpu = get_cpu();
 110         put_cpu();
 111
 112         tracing_start_cmdline_record();
 113         tracing_start_function_trace(tr);
 114         return 0;
 115 }
 116
 117 static void function_trace_reset(struct trace_array *tr)
 118 {
 119         tracing_stop_function_trace(tr);
 120         tracing_stop_cmdline_record();
 121 }
 122
 123 static void function_trace_start(struct trace_array *tr)
 124 {
 125         tracing_reset_online_cpus(&tr->trace_buffer);
 126 }
 127
 128 static void
 129 function_trace_call(unsigned long ip, unsigned long parent_ip,
 130                     struct ftrace_ops *op, struct pt_regs *pt_regs)
 131 {
 132         struct trace_array *tr = op->private;
 133         struct trace_array_cpu *data;
 134         unsigned long flags;
 135         int bit;
 136         int cpu;
 137         int pc;
 138
 139         if (unlikely(!tr->function_enabled))
 140                 return;
 141
 142         pc = preempt_count();
 143         preempt_disable_notrace();
 144
 145         bit = trace_test_and_set_recursion(TRACE_FTRACE_START, TRACE_FTRACE_MAX);
 146         if (bit < 0)
 147                 goto out;
 148
 149         cpu = smp_processor_id();
 150         data = per_cpu_ptr(tr->trace_buffer.data, cpu);
 151         if (!atomic_read(&data->disabled)) {
 152                 local_save_flags(flags);
 153                 trace_function(tr, ip, parent_ip, flags, pc);
 154         }
 155         trace_clear_recursion(bit);
 156
 157  out:
 158         preempt_enable_notrace();
 159 }
 160
 161 static void
 162 function_stack_trace_call(unsigned long ip, unsigned long parent_ip,
 163                           struct ftrace_ops *op, struct pt_regs *pt_regs)
 164 {
 165         struct trace_array *tr = op->private;
 166         struct trace_array_cpu *data;
 167         unsigned long flags;
 168         long disabled;
 169         int cpu;
 170         int pc;
 171
 172         if (unlikely(!tr->function_enabled))
 173                 return;
 174
 175         /*
 176          * Need to use raw, since this must be called before the
 177          * recursive protection is performed.
 178          */
 179         local_irq_save(flags);
 180         cpu = raw_smp_processor_id();
 181         data = per_cpu_ptr(tr->trace_buffer.data, cpu);
 182         disabled = atomic_inc_return(&data->disabled);
 183
 184         if (likely(disabled == 1)) {
 185                 pc = preempt_count();
 186                 trace_function(tr, ip, parent_ip, flags, pc);
 187                 /*
 188                  * skip over 5 funcs:
 189                  *    __ftrace_trace_stack,
 190                  *    __trace_stack,
 191                  *    function_stack_trace_call
 192                  *    ftrace_list_func
 193                  *    ftrace_call
 194                  */
 195                 __trace_stack(tr, flags, 5, pc);
 196         }
 197
 198         atomic_dec(&data->disabled);
 199         local_irq_restore(flags);
 200 }
 201
 202 static struct ftrace_ops trace_ops __read_mostly =
 203 {
 204         .func = function_trace_call,
 205         .flags = FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_RECURSION_SAFE,
 206 };
 207
 208 static struct ftrace_ops trace_stack_ops __read_mostly =
 209 {
 210         .func = function_stack_trace_call,
 211         .flags = FTRACE_OPS_FL_GLOBAL | FTRACE_OPS_FL_RECURSION_SAFE,
 212 };
 213
 214 static struct tracer_opt func_opts[] = {
 215 #ifdef CONFIG_STACKTRACE
 216         { TRACER_OPT(func_stack_trace, TRACE_FUNC_OPT_STACK) },
 217 #endif
 218         { } /* Always set a last empty entry */
 219 };
 220
 221 static struct tracer_flags func_flags = {
 222         .val = 0, /* By default: all flags disabled */
 223         .opts = func_opts
 224 };
 225
 226 static void tracing_start_function_trace(struct trace_array *tr)
 227 {
 228         tr->function_enabled = 0;
 229         register_ftrace_function(tr->ops);
 230         tr->function_enabled = 1;
 231 }
 232
 233 static void tracing_stop_function_trace(struct trace_array *tr)
 234 {
 235         tr->function_enabled = 0;
 236         unregister_ftrace_function(tr->ops);
 237 }
 238
 239 static int
 240 func_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
 241 {
 242         switch (bit) {
 243         case TRACE_FUNC_OPT_STACK:
 244                 /* do nothing if already set */
 245                 if (!!set == !!(func_flags.val & TRACE_FUNC_OPT_STACK))
 246                         break;
 247
 248                 unregister_ftrace_function(tr->ops);
 249
 250                 if (set) {
 251                         tr->ops = &trace_stack_ops;
 252                         register_ftrace_function(tr->ops);
 253                 } else {
 254                         tr->ops = &trace_ops;
 255                         register_ftrace_function(tr->ops);
 256                 }
 257
 258                 break;
 259         default:
 260                 return -EINVAL;
 261         }
 262
 263         return 0;
 264 }
 265
 266 static struct tracer function_trace __tracer_data =
 267 {
 268         .name           = "function",
 269         .init           = function_trace_init,
 270         .reset          = function_trace_reset,
 271         .start          = function_trace_start,
 272         .wait_pipe      = poll_wait_pipe,
 273         .flags          = &func_flags,
 274         .set_flag       = func_set_flag,
 275         .allow_instances = true,
 276 #ifdef CONFIG_FTRACE_SELFTEST
 277         .selftest       = trace_selftest_startup_function,
 278 #endif
 279 };
 280
 281 #ifdef CONFIG_DYNAMIC_FTRACE
 282 static int update_count(void **data)
 283 {
 284         unsigned long *count = (long *)data;
 285
 286         if (!*count)
 287                 return 0;
 288
 289         if (*count != -1)
 290                 (*count)--;
 291
 292         return 1;
 293 }
 294
 295 static void
 296 ftrace_traceon_count(unsigned long ip, unsigned long parent_ip, void **data)
 297 {
 298         if (tracing_is_on())
 299                 return;
 300
 301         if (update_count(data))
 302                 tracing_on();
 303 }
 304
 305 static void
 306 ftrace_traceoff_count(unsigned long ip, unsigned long parent_ip, void **data)
 307 {
 308         if (!tracing_is_on())
 309                 return;
 310
 311         if (update_count(data))
 312                 tracing_off();
 313 }
 314
 315 static void
 316 ftrace_traceon(unsigned long ip, unsigned long parent_ip, void **data)
 317 {
 318         if (tracing_is_on())
 319                 return;
 320
 321         tracing_on();
 322 }
 323
 324 static void
 325 ftrace_traceoff(unsigned long ip, unsigned long parent_ip, void **data)
 326 {
 327         if (!tracing_is_on())
 328                 return;
 329
 330         tracing_off();
 331 }
 332
 333 /*
 334  * Skip 4:
 335  *   ftrace_stacktrace()
 336  *   function_trace_probe_call()
 337  *   ftrace_ops_list_func()
 338  *   ftrace_call()
 339  */
 340 #define STACK_SKIP 4
 341
 342 static void
 343 ftrace_stacktrace(unsigned long ip, unsigned long parent_ip, void **data)
 344 {
 345         trace_dump_stack(STACK_SKIP);
 346 }
 347
 348 static void
 349 ftrace_stacktrace_count(unsigned long ip, unsigned long parent_ip, void **data)
 350 {
 351         if (!tracing_is_on())
 352                 return;
 353
 354         if (update_count(data))
 355                 trace_dump_stack(STACK_SKIP);
 356 }
 357
 358 static void
 359 ftrace_dump_probe(unsigned long ip, unsigned long parent_ip, void **data)
 360 {
 361         if (update_count(data))
 362                 ftrace_dump(DUMP_ALL);
 363 }
 364
 365 /* Only dump the current CPU buffer. */
 366 static void
 367 ftrace_cpudump_probe(unsigned long ip, unsigned long parent_ip, void **data)
 368 {
 369         if (update_count(data))
 370                 ftrace_dump(DUMP_ORIG);
 371 }
 372
 373 static int
 374 ftrace_probe_print(const char *name, struct seq_file *m,
 375                    unsigned long ip, void *data)
 376 {
 377         long count = (long)data;
 378
 379         seq_printf(m, "%ps:%s", (void *)ip, name);
 380
 381         if (count == -1)
 382                 seq_printf(m, ":unlimited\n");
 383         else
 384                 seq_printf(m, ":count=%ld\n", count);
 385
 386         return 0;
 387 }
 388
 389 static int
 390 ftrace_traceon_print(struct seq_file *m, unsigned long ip,
 391                          struct ftrace_probe_ops *ops, void *data)
 392 {
 393         return ftrace_probe_print("traceon", m, ip, data);
 394 }
 395
 396 static int
 397 ftrace_traceoff_print(struct seq_file *m, unsigned long ip,
 398                          struct ftrace_probe_ops *ops, void *data)
 399 {
 400         return ftrace_probe_print("traceoff", m, ip, data);
 401 }
 402
 403 static int
 404 ftrace_stacktrace_print(struct seq_file *m, unsigned long ip,
 405                         struct ftrace_probe_ops *ops, void *data)
 406 {
 407         return ftrace_probe_print("stacktrace", m, ip, data);
 408 }
 409
 410 static int
 411 ftrace_dump_print(struct seq_file *m, unsigned long ip,
 412                         struct ftrace_probe_ops *ops, void *data)
 413 {
 414         return ftrace_probe_print("dump", m, ip, data);
 415 }
 416
 417 static int
 418 ftrace_cpudump_print(struct seq_file *m, unsigned long ip,
 419                         struct ftrace_probe_ops *ops, void *data)
 420 {
 421         return ftrace_probe_print("cpudump", m, ip, data);
 422 }
 423
 424 static struct ftrace_probe_ops traceon_count_probe_ops = {
 425         .func                   = ftrace_traceon_count,
 426         .print                  = ftrace_traceon_print,
 427 };
 428
 429 static struct ftrace_probe_ops traceoff_count_probe_ops = {
 430         .func                   = ftrace_traceoff_count,
 431         .print                  = ftrace_traceoff_print,
 432 };
 433
 434 static struct ftrace_probe_ops stacktrace_count_probe_ops = {
 435         .func                   = ftrace_stacktrace_count,
 436         .print                  = ftrace_stacktrace_print,
 437 };
 438
 439 static struct ftrace_probe_ops dump_probe_ops = {
 440         .func                   = ftrace_dump_probe,
 441         .print                  = ftrace_dump_print,
 442 };
 443
 444 static struct ftrace_probe_ops cpudump_probe_ops = {
 445         .func                   = ftrace_cpudump_probe,
 446         .print                  = ftrace_cpudump_print,
 447 };
 448
 449 static struct ftrace_probe_ops traceon_probe_ops = {
 450         .func                   = ftrace_traceon,
 451         .print                  = ftrace_traceon_print,
 452 };
 453
 454 static struct ftrace_probe_ops traceoff_probe_ops = {
 455         .func                   = ftrace_traceoff,
 456         .print                  = ftrace_traceoff_print,
 457 };
 458
 459 static struct ftrace_probe_ops stacktrace_probe_ops = {
 460         .func                   = ftrace_stacktrace,
 461         .print                  = ftrace_stacktrace_print,
 462 };
 463
 464 static int
 465 ftrace_trace_probe_callback(struct ftrace_probe_ops *ops,
 466                             struct ftrace_hash *hash, char *glob,
 467                             char *cmd, char *param, int enable)
 468 {
 469         void *count = (void *)-1;
 470         char *number;
 471         int ret;
 472
 473         /* hash funcs only work with set_ftrace_filter */
 474         if (!enable)
 475                 return -EINVAL;
 476
 477         if (glob[0] == '!') {
 478                 unregister_ftrace_function_probe_func(glob+1, ops);
 479                 return 0;
 480         }
 481
 482         if (!param)
 483                 goto out_reg;
 484
 485         number = strsep(&param, ":");
 486
 487         if (!strlen(number))
 488                 goto out_reg;
 489
 490         /*
 491          * We use the callback data field (which is a pointer)
 492          * as our counter.
 493          */
 494         ret = kstrtoul(number, 0, (unsigned long *)&count);
 495         if (ret)
 496                 return ret;
 497
 498  out_reg:
 499         ret = register_ftrace_function_probe(glob, ops, count);
 500
 501         return ret < 0 ? ret : 0;
 502 }
 503
 504 static int
 505 ftrace_trace_onoff_callback(struct ftrace_hash *hash,
 506                             char *glob, char *cmd, char *param, int enable)
 507 {
 508         struct ftrace_probe_ops *ops;
 509
 510         /* we register both traceon and traceoff to this callback */
 511         if (strcmp(cmd, "traceon") == 0)
 512                 ops = param ? &traceon_count_probe_ops : &traceon_probe_ops;
 513         else
 514                 ops = param ? &traceoff_count_probe_ops : &traceoff_probe_ops;
 515
 516         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 517                                            param, enable);
 518 }
 519
 520 static int
 521 ftrace_stacktrace_callback(struct ftrace_hash *hash,
 522                            char *glob, char *cmd, char *param, int enable)
 523 {
 524         struct ftrace_probe_ops *ops;
 525
 526         ops = param ? &stacktrace_count_probe_ops : &stacktrace_probe_ops;
 527
 528         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 529                                            param, enable);
 530 }
 531
 532 static int
 533 ftrace_dump_callback(struct ftrace_hash *hash,
 534                            char *glob, char *cmd, char *param, int enable)
 535 {
 536         struct ftrace_probe_ops *ops;
 537
 538         ops = &dump_probe_ops;
 539
 540         /* Only dump once. */
 541         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 542                                            "1", enable);
 543 }
 544
 545 static int
 546 ftrace_cpudump_callback(struct ftrace_hash *hash,
 547                            char *glob, char *cmd, char *param, int enable)
 548 {
 549         struct ftrace_probe_ops *ops;
 550
 551         ops = &cpudump_probe_ops;
 552
 553         /* Only dump once. */
 554         return ftrace_trace_probe_callback(ops, hash, glob, cmd,
 555                                            "1", enable);
 556 }
 557
 558 static struct ftrace_func_command ftrace_traceon_cmd = {
 559         .name                   = "traceon",
 560         .func                   = ftrace_trace_onoff_callback,
 561 };
 562
 563 static struct ftrace_func_command ftrace_traceoff_cmd = {
 564         .name                   = "traceoff",
 565         .func                   = ftrace_trace_onoff_callback,
 566 };
 567
 568 static struct ftrace_func_command ftrace_stacktrace_cmd = {
 569         .name                   = "stacktrace",
 570         .func                   = ftrace_stacktrace_callback,
 571 };
 572
 573 static struct ftrace_func_command ftrace_dump_cmd = {
 574         .name                   = "dump",
 575         .func                   = ftrace_dump_callback,
 576 };
 577
 578 static struct ftrace_func_command ftrace_cpudump_cmd = {
 579         .name                   = "cpudump",
 580         .func                   = ftrace_cpudump_callback,
 581 };
 582
 583 static int __init init_func_cmd_traceon(void)
 584 {
 585         int ret;
 586
 587         ret = register_ftrace_command(&ftrace_traceoff_cmd);
 588         if (ret)
 589                 return ret;
 590
 591         ret = register_ftrace_command(&ftrace_traceon_cmd);
 592         if (ret)
 593                 goto out_free_traceoff;
 594
 595         ret = register_ftrace_command(&ftrace_stacktrace_cmd);
 596         if (ret)
 597                 goto out_free_traceon;
 598
 599         ret = register_ftrace_command(&ftrace_dump_cmd);
 600         if (ret)
 601                 goto out_free_stacktrace;
 602
 603         ret = register_ftrace_command(&ftrace_cpudump_cmd);
 604         if (ret)
 605                 goto out_free_dump;
 606
 607         return 0;
 608
 609  out_free_dump:
 610         unregister_ftrace_command(&ftrace_dump_cmd);
 611  out_free_stacktrace:
 612         unregister_ftrace_command(&ftrace_stacktrace_cmd);
 613  out_free_traceon:
 614         unregister_ftrace_command(&ftrace_traceon_cmd);
 615  out_free_traceoff:
 616         unregister_ftrace_command(&ftrace_traceoff_cmd);
 617
 618         return ret;
 619 }
 620 #else
 621 static inline int init_func_cmd_traceon(void)
 622 {
 623         return 0;
 624 }
 625 #endif /* CONFIG_DYNAMIC_FTRACE */
 626
 627 static __init int init_function_trace(void)
 628 {
 629         init_func_cmd_traceon();
 630         return register_tracer(&function_trace);
 631 }
 632 core_initcall(init_function_trace);