2 * Code for replacing ftrace calls with jumps.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Thanks goes to Ingo Molnar, for suggesting the idea.
7 * Mathieu Desnoyers, for suggesting postponing the modifications.
8 * Arjan van de Ven, for keeping me straight, and explaining to me
9 * the dangers of modifying code on the run.
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/spinlock.h>
15 #include <linux/hardirq.h>
16 #include <linux/uaccess.h>
17 #include <linux/ftrace.h>
18 #include <linux/percpu.h>
19 #include <linux/sched.h>
20 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/module.h>
24 #include <trace/syscall.h>
26 #include <asm/cacheflush.h>
27 #include <asm/kprobes.h>
28 #include <asm/ftrace.h>
31 #ifdef CONFIG_DYNAMIC_FTRACE
33 int ftrace_arch_code_modify_prepare(void)
36 set_all_modules_text_rw();
40 int ftrace_arch_code_modify_post_process(void)
42 set_all_modules_text_ro();
47 union ftrace_code_union {
48 char code[MCOUNT_INSN_SIZE];
52 } __attribute__((packed));
55 static int ftrace_calc_offset(long ip, long addr)
57 return (int)(addr - ip);
60 static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
62 static union ftrace_code_union calc;
65 calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
68 * No locking needed, this must be called via kstop_machine
69 * which in essence is like running on a uniprocessor machine.
75 within(unsigned long addr, unsigned long start, unsigned long end)
77 return addr >= start && addr < end;
81 do_ftrace_mod_code(unsigned long ip, const void *new_code)
84 * On x86_64, kernel text mappings are mapped read-only with
85 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
86 * of the kernel text mapping to modify the kernel text.
88 * For 32bit kernels, these mappings are same and we can use
89 * kernel identity mapping to modify code.
91 if (within(ip, (unsigned long)_text, (unsigned long)_etext))
92 ip = (unsigned long)__va(__pa(ip));
94 return probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE);
97 static const unsigned char *ftrace_nop_replace(void)
99 return ideal_nops[NOP_ATOMIC5];
103 ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
104 unsigned const char *new_code)
106 unsigned char replaced[MCOUNT_INSN_SIZE];
109 * Note: Due to modules and __init, code can
110 * disappear and change, we need to protect against faulting
111 * as well as code changing. We do this by using the
112 * probe_kernel_* functions.
114 * No real locking needed, this code is run through
115 * kstop_machine, or before SMP starts.
118 /* read the text we want to modify */
119 if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
122 /* Make sure it is what we expect it to be */
123 if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
126 /* replace the text with the new text */
127 if (do_ftrace_mod_code(ip, new_code))
135 int ftrace_make_nop(struct module *mod,
136 struct dyn_ftrace *rec, unsigned long addr)
138 unsigned const char *new, *old;
139 unsigned long ip = rec->ip;
141 old = ftrace_call_replace(ip, addr);
142 new = ftrace_nop_replace();
144 return ftrace_modify_code(rec->ip, old, new);
147 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
149 unsigned const char *new, *old;
150 unsigned long ip = rec->ip;
152 old = ftrace_nop_replace();
153 new = ftrace_call_replace(ip, addr);
155 return ftrace_modify_code(rec->ip, old, new);
158 int ftrace_update_ftrace_func(ftrace_func_t func)
160 unsigned long ip = (unsigned long)(&ftrace_call);
161 unsigned char old[MCOUNT_INSN_SIZE], *new;
164 memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
165 new = ftrace_call_replace(ip, (unsigned long)func);
166 ret = ftrace_modify_code(ip, old, new);
171 int modifying_ftrace_code __read_mostly;
174 * A breakpoint was added to the code address we are about to
175 * modify, and this is the handle that will just skip over it.
176 * We are either changing a nop into a trace call, or a trace
177 * call to a nop. While the change is taking place, we treat
178 * it just like it was a nop.
180 int ftrace_int3_handler(struct pt_regs *regs)
182 if (WARN_ON_ONCE(!regs))
185 if (!ftrace_location(regs->ip - 1))
188 regs->ip += MCOUNT_INSN_SIZE - 1;
193 static int ftrace_write(unsigned long ip, const char *val, int size)
196 * On x86_64, kernel text mappings are mapped read-only with
197 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
198 * of the kernel text mapping to modify the kernel text.
200 * For 32bit kernels, these mappings are same and we can use
201 * kernel identity mapping to modify code.
203 if (within(ip, (unsigned long)_text, (unsigned long)_etext))
204 ip = (unsigned long)__va(__pa(ip));
206 return probe_kernel_write((void *)ip, val, size);
209 static int add_break(unsigned long ip, const char *old)
211 unsigned char replaced[MCOUNT_INSN_SIZE];
212 unsigned char brk = BREAKPOINT_INSTRUCTION;
214 if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
217 /* Make sure it is what we expect it to be */
218 if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
221 if (ftrace_write(ip, &brk, 1))
227 static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
229 unsigned const char *old;
230 unsigned long ip = rec->ip;
232 old = ftrace_call_replace(ip, addr);
234 return add_break(rec->ip, old);
238 static int add_brk_on_nop(struct dyn_ftrace *rec)
240 unsigned const char *old;
242 old = ftrace_nop_replace();
244 return add_break(rec->ip, old);
247 static int add_breakpoints(struct dyn_ftrace *rec, int enable)
249 unsigned long ftrace_addr;
252 ret = ftrace_test_record(rec, enable);
254 ftrace_addr = (unsigned long)FTRACE_ADDR;
257 case FTRACE_UPDATE_IGNORE:
260 case FTRACE_UPDATE_MAKE_CALL:
261 /* converting nop to call */
262 return add_brk_on_nop(rec);
264 case FTRACE_UPDATE_MAKE_NOP:
265 /* converting a call to a nop */
266 return add_brk_on_call(rec, ftrace_addr);
272 * On error, we need to remove breakpoints. This needs to
273 * be done caefully. If the address does not currently have a
274 * breakpoint, we know we are done. Otherwise, we look at the
275 * remaining 4 bytes of the instruction. If it matches a nop
276 * we replace the breakpoint with the nop. Otherwise we replace
277 * it with the call instruction.
279 static int remove_breakpoint(struct dyn_ftrace *rec)
281 unsigned char ins[MCOUNT_INSN_SIZE];
282 unsigned char brk = BREAKPOINT_INSTRUCTION;
283 const unsigned char *nop;
284 unsigned long ftrace_addr;
285 unsigned long ip = rec->ip;
287 /* If we fail the read, just give up */
288 if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
291 /* If this does not have a breakpoint, we are done */
295 nop = ftrace_nop_replace();
298 * If the last 4 bytes of the instruction do not match
299 * a nop, then we assume that this is a call to ftrace_addr.
301 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
303 * For extra paranoidism, we check if the breakpoint is on
304 * a call that would actually jump to the ftrace_addr.
305 * If not, don't touch the breakpoint, we make just create
308 ftrace_addr = (unsigned long)FTRACE_ADDR;
309 nop = ftrace_call_replace(ip, ftrace_addr);
311 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
315 return probe_kernel_write((void *)ip, &nop[0], 1);
318 static int add_update_code(unsigned long ip, unsigned const char *new)
320 /* skip breakpoint */
323 if (ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1))
328 static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
330 unsigned long ip = rec->ip;
331 unsigned const char *new;
333 new = ftrace_call_replace(ip, addr);
334 return add_update_code(ip, new);
337 static int add_update_nop(struct dyn_ftrace *rec)
339 unsigned long ip = rec->ip;
340 unsigned const char *new;
342 new = ftrace_nop_replace();
343 return add_update_code(ip, new);
346 static int add_update(struct dyn_ftrace *rec, int enable)
348 unsigned long ftrace_addr;
351 ret = ftrace_test_record(rec, enable);
353 ftrace_addr = (unsigned long)FTRACE_ADDR;
356 case FTRACE_UPDATE_IGNORE:
359 case FTRACE_UPDATE_MAKE_CALL:
360 /* converting nop to call */
361 return add_update_call(rec, ftrace_addr);
363 case FTRACE_UPDATE_MAKE_NOP:
364 /* converting a call to a nop */
365 return add_update_nop(rec);
371 static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
373 unsigned long ip = rec->ip;
374 unsigned const char *new;
376 new = ftrace_call_replace(ip, addr);
378 if (ftrace_write(ip, new, 1))
384 static int finish_update_nop(struct dyn_ftrace *rec)
386 unsigned long ip = rec->ip;
387 unsigned const char *new;
389 new = ftrace_nop_replace();
391 if (ftrace_write(ip, new, 1))
396 static int finish_update(struct dyn_ftrace *rec, int enable)
398 unsigned long ftrace_addr;
401 ret = ftrace_update_record(rec, enable);
403 ftrace_addr = (unsigned long)FTRACE_ADDR;
406 case FTRACE_UPDATE_IGNORE:
409 case FTRACE_UPDATE_MAKE_CALL:
410 /* converting nop to call */
411 return finish_update_call(rec, ftrace_addr);
413 case FTRACE_UPDATE_MAKE_NOP:
414 /* converting a call to a nop */
415 return finish_update_nop(rec);
421 static void do_sync_core(void *data)
426 static void run_sync(void)
428 int enable_irqs = irqs_disabled();
430 /* We may be called with interrupts disbled (on bootup). */
433 on_each_cpu(do_sync_core, NULL, 1);
438 void ftrace_replace_code(int enable)
440 struct ftrace_rec_iter *iter;
441 struct dyn_ftrace *rec;
442 const char *report = "adding breakpoints";
446 for_ftrace_rec_iter(iter) {
447 rec = ftrace_rec_iter_record(iter);
449 ret = add_breakpoints(rec, enable);
451 goto remove_breakpoints;
457 report = "updating code";
459 for_ftrace_rec_iter(iter) {
460 rec = ftrace_rec_iter_record(iter);
462 ret = add_update(rec, enable);
464 goto remove_breakpoints;
469 report = "removing breakpoints";
471 for_ftrace_rec_iter(iter) {
472 rec = ftrace_rec_iter_record(iter);
474 ret = finish_update(rec, enable);
476 goto remove_breakpoints;
484 ftrace_bug(ret, rec ? rec->ip : 0);
485 printk(KERN_WARNING "Failed on %s (%d):\n", report, count);
486 for_ftrace_rec_iter(iter) {
487 rec = ftrace_rec_iter_record(iter);
488 remove_breakpoint(rec);
492 void arch_ftrace_update_code(int command)
494 modifying_ftrace_code++;
496 ftrace_modify_all_code(command);
498 modifying_ftrace_code--;
501 int __init ftrace_dyn_arch_init(void *data)
503 /* The return code is retured via data */
504 *(unsigned long *)data = 0;
510 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
512 #ifdef CONFIG_DYNAMIC_FTRACE
513 extern void ftrace_graph_call(void);
515 static int ftrace_mod_jmp(unsigned long ip,
516 int old_offset, int new_offset)
518 unsigned char code[MCOUNT_INSN_SIZE];
520 if (probe_kernel_read(code, (void *)ip, MCOUNT_INSN_SIZE))
523 if (code[0] != 0xe9 || old_offset != *(int *)(&code[1]))
526 *(int *)(&code[1]) = new_offset;
528 if (do_ftrace_mod_code(ip, &code))
534 int ftrace_enable_ftrace_graph_caller(void)
536 unsigned long ip = (unsigned long)(&ftrace_graph_call);
537 int old_offset, new_offset;
539 old_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
540 new_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
542 return ftrace_mod_jmp(ip, old_offset, new_offset);
545 int ftrace_disable_ftrace_graph_caller(void)
547 unsigned long ip = (unsigned long)(&ftrace_graph_call);
548 int old_offset, new_offset;
550 old_offset = (unsigned long)(&ftrace_graph_caller) - (ip + MCOUNT_INSN_SIZE);
551 new_offset = (unsigned long)(&ftrace_stub) - (ip + MCOUNT_INSN_SIZE);
553 return ftrace_mod_jmp(ip, old_offset, new_offset);
556 #endif /* !CONFIG_DYNAMIC_FTRACE */
559 * Hook the return address and push it in the stack of return addrs
560 * in current thread info.
562 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
563 unsigned long frame_pointer)
567 struct ftrace_graph_ent trace;
568 unsigned long return_hooker = (unsigned long)
571 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
575 * Protect against fault, even if it shouldn't
576 * happen. This tool is too much intrusive to
577 * ignore such a protection.
580 "1: " _ASM_MOV " (%[parent]), %[old]\n"
581 "2: " _ASM_MOV " %[return_hooker], (%[parent])\n"
582 " movl $0, %[faulted]\n"
585 ".section .fixup, \"ax\"\n"
586 "4: movl $1, %[faulted]\n"
593 : [old] "=&r" (old), [faulted] "=r" (faulted)
594 : [parent] "r" (parent), [return_hooker] "r" (return_hooker)
598 if (unlikely(faulted)) {
604 trace.func = self_addr;
605 trace.depth = current->curr_ret_stack + 1;
607 /* Only trace if the calling function expects to */
608 if (!ftrace_graph_entry(&trace)) {
613 if (ftrace_push_return_trace(old, self_addr, &trace.depth,
614 frame_pointer) == -EBUSY) {
619 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */