2 * Code for replacing ftrace calls with jumps.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2009 DSLab, Lanzhou University, China
6 * Author: Wu Zhangjin <wuzj@lemote.com>
8 * Thanks goes to Steven Rostedt for writing the original x86 version.
11 #include <linux/uaccess.h>
12 #include <linux/init.h>
13 #include <linux/ftrace.h>
15 #include <asm/cacheflush.h>
17 #include <asm/asm-offsets.h>
19 #ifdef CONFIG_DYNAMIC_FTRACE
21 #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */
22 #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */
23 #define jump_insn_encode(op_code, addr) \
24 ((unsigned int)((op_code) | (((addr) >> 2) & ADDR_MASK)))
26 static unsigned int ftrace_nop = 0x00000000;
28 static int ftrace_modify_code(unsigned long ip, unsigned int new_code)
30 *(unsigned int *)ip = new_code;
32 flush_icache_range(ip, ip + 8);
38 static int jal_mcount;
40 int ftrace_make_nop(struct module *mod,
41 struct dyn_ftrace *rec, unsigned long addr)
44 unsigned long ip = rec->ip;
46 /* We have compiled module with -mlong-calls, but compiled the kernel
47 * without it, we need to cope with them respectively. */
48 if (ip & 0x40000000) {
49 /* record it for ftrace_make_call */
51 lui_v1 = *(unsigned int *)ip;
53 /* lui v1, hi_16bit_of_mcount --> b 1f (0x10000004)
54 * addiu v1, v1, low_16bit_of_mcount
62 /* record/calculate it for ftrace_make_call */
63 if (jal_mcount == 0) {
64 /* We can record it directly like this:
65 * jal_mcount = *(unsigned int *)ip;
66 * Herein, jump over the first two nop instructions */
67 jal_mcount = jump_insn_encode(JAL, (MCOUNT_ADDR + 8));
75 return ftrace_modify_code(ip, new);
78 static int modified; /* initialized as 0 by default */
80 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
83 unsigned long ip = rec->ip;
85 /* We just need to remove the "b ftrace_stub" at the fist time! */
88 ftrace_modify_code(addr, ftrace_nop);
90 /* ip, module: 0xc0000000, kernel: 0x80000000 */
91 new = (ip & 0x40000000) ? lui_v1 : jal_mcount;
93 return ftrace_modify_code(ip, new);
96 #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call))
98 int ftrace_update_ftrace_func(ftrace_func_t func)
102 new = jump_insn_encode(JAL, (unsigned long)func);
104 return ftrace_modify_code(FTRACE_CALL_IP, new);
107 int __init ftrace_dyn_arch_init(void *data)
109 /* The return code is retured via data */
110 *(unsigned long *)data = 0;
114 #endif /* CONFIG_DYNAMIC_FTRACE */
116 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
118 #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */
119 #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */
120 #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */
122 unsigned long ftrace_get_parent_addr(unsigned long self_addr,
123 unsigned long parent,
124 unsigned long parent_addr,
127 unsigned long sp, ip, ra;
130 /* in module or kernel? */
131 if (self_addr & 0x40000000) {
132 /* module: move to the instruction "lui v1, HI_16BIT_OF_MCOUNT" */
135 /* kernel: move to the instruction "move ra, at" */
139 /* search the text until finding the non-store instruction or "s{d,w}
140 * ra, offset(sp)" instruction */
144 /* get the code at "ip" */
145 code = *(unsigned int *)ip;
147 /* If we hit the non-store instruction before finding where the
148 * ra is stored, then this is a leaf function and it does not
149 * store the ra on the stack. */
150 if ((code & S_R_SP) != S_R_SP)
153 } while (((code & S_RA_SP) != S_RA_SP));
155 sp = fp + (code & OFFSET_MASK);
156 ra = *(unsigned long *)sp;
165 * Hook the return address and push it in the stack of return addrs
166 * in current thread info.
168 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
172 struct ftrace_graph_ent trace;
173 unsigned long return_hooker = (unsigned long)
176 if (unlikely(atomic_read(¤t->tracing_graph_pause)))
179 /* "parent" is the stack address saved the return address of the caller
180 * of _mcount, for a leaf function not save the return address in the
181 * stack address, so, we "emulate" one in _mcount's stack space, and
182 * hijack it directly, but for a non-leaf function, it will save the
183 * return address to the its stack space, so, we can not hijack the
184 * "parent" directly, but need to find the real stack address,
185 * ftrace_get_parent_addr() does it!
190 parent = (unsigned long *)ftrace_get_parent_addr(self_addr, old,
191 (unsigned long)parent,
194 /* If fails when getting the stack address of the non-leaf function's
195 * ra, stop function graph tracer and return */
202 *parent = return_hooker;
204 if (ftrace_push_return_trace(old, self_addr, &trace.depth, fp) ==
210 trace.func = self_addr;
212 /* Only trace if the calling function expects to */
213 if (!ftrace_graph_entry(&trace)) {
214 current->curr_ret_stack--;
218 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */