2 * Copyright (C) 2014 Steven Rostedt, Red Hat Inc
5 #include <linux/linkage.h>
6 #include <asm/ptrace.h>
7 #include <asm/ftrace.h>
8 #include <asm/export.h>
12 .section .entry.text, "ax"
14 #ifdef CC_USING_FENTRY
15 # define function_hook __fentry__
16 EXPORT_SYMBOL(__fentry__)
18 # define function_hook mcount
22 /* All cases save the original rbp (8 bytes) */
23 #ifdef CONFIG_FRAME_POINTER
24 # ifdef CC_USING_FENTRY
25 /* Save parent and function stack frames (rip and rbp) */
26 # define MCOUNT_FRAME_SIZE (8+16*2)
28 /* Save just function stack frame (rip and rbp) */
29 # define MCOUNT_FRAME_SIZE (8+16)
32 /* No need to save a stack frame */
33 # define MCOUNT_FRAME_SIZE 8
34 #endif /* CONFIG_FRAME_POINTER */
36 /* Size of stack used to save mcount regs in save_mcount_regs */
37 #define MCOUNT_REG_SIZE (SS+8 + MCOUNT_FRAME_SIZE)
40 * gcc -pg option adds a call to 'mcount' in most functions.
41 * When -mfentry is used, the call is to 'fentry' and not 'mcount'
42 * and is done before the function's stack frame is set up.
43 * They both require a set of regs to be saved before calling
44 * any C code and restored before returning back to the function.
46 * On boot up, all these calls are converted into nops. When tracing
47 * is enabled, the call can jump to either ftrace_caller or
48 * ftrace_regs_caller. Callbacks (tracing functions) that require
49 * ftrace_regs_caller (like kprobes) need to have pt_regs passed to
50 * it. For this reason, the size of the pt_regs structure will be
51 * allocated on the stack and the required mcount registers will
52 * be saved in the locations that pt_regs has them in.
56 * @added: the amount of stack added before calling this
58 * After this is called, the following registers contain:
60 * %rdi - holds the address that called the trampoline
61 * %rsi - holds the parent function (traced function's return address)
62 * %rdx - holds the original %rbp
64 .macro save_mcount_regs added=0
66 /* Always save the original rbp */
69 #ifdef CONFIG_FRAME_POINTER
71 * Stack traces will stop at the ftrace trampoline if the frame pointer
72 * is not set up properly. If fentry is used, we need to save a frame
73 * pointer for the parent as well as the function traced, because the
74 * fentry is called before the stack frame is set up, where as mcount
75 * is called afterward.
77 #ifdef CC_USING_FENTRY
78 /* Save the parent pointer (skip orig rbp and our return address) */
79 pushq \added+8*2(%rsp)
82 /* Save the return address (now skip orig rbp, rbp and parent) */
83 pushq \added+8*3(%rsp)
85 /* Can't assume that rip is before this (unless added was zero) */
90 #endif /* CONFIG_FRAME_POINTER */
93 * We add enough stack to save all regs.
95 subq $(MCOUNT_REG_SIZE - MCOUNT_FRAME_SIZE), %rsp
104 * Save the original RBP. Even though the mcount ABI does not
105 * require this, it helps out callers.
107 movq MCOUNT_REG_SIZE-8(%rsp), %rdx
110 /* Copy the parent address into %rsi (second parameter) */
111 #ifdef CC_USING_FENTRY
112 movq MCOUNT_REG_SIZE+8+\added(%rsp), %rsi
114 /* %rdx contains original %rbp */
118 /* Move RIP to its proper location */
119 movq MCOUNT_REG_SIZE+\added(%rsp), %rdi
123 * Now %rdi (the first parameter) has the return address of
124 * where ftrace_call returns. But the callbacks expect the
125 * address of the call itself.
127 subq $MCOUNT_INSN_SIZE, %rdi
130 .macro restore_mcount_regs
139 /* ftrace_regs_caller can modify %rbp */
142 addq $MCOUNT_REG_SIZE, %rsp
146 #ifdef CONFIG_DYNAMIC_FTRACE
153 /* save_mcount_regs fills in first two parameters */
156 GLOBAL(ftrace_caller_op_ptr)
157 /* Load the ftrace_ops into the 3rd parameter */
158 movq function_trace_op(%rip), %rdx
160 /* regs go into 4th parameter (but make it NULL) */
169 * The copied trampoline must call ftrace_epilogue as it
170 * still may need to call the function graph tracer.
172 * The code up to this label is copied into trampolines so
173 * think twice before adding any new code or changing the
176 GLOBAL(ftrace_epilogue)
178 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
179 GLOBAL(ftrace_graph_call)
183 /* This is weak to keep gas from relaxing the jumps */
188 ENTRY(ftrace_regs_caller)
189 /* Save the current flags before any operations that can change them */
192 /* added 8 bytes to save flags */
194 /* save_mcount_regs fills in first two parameters */
196 GLOBAL(ftrace_regs_caller_op_ptr)
197 /* Load the ftrace_ops into the 3rd parameter */
198 movq function_trace_op(%rip), %rdx
200 /* Save the rest of pt_regs */
208 /* Copy saved flags */
209 movq MCOUNT_REG_SIZE(%rsp), %rcx
210 movq %rcx, EFLAGS(%rsp)
211 /* Kernel segments */
212 movq $__KERNEL_DS, %rcx
214 movq $__KERNEL_CS, %rcx
216 /* Stack - skipping return address and flags */
217 leaq MCOUNT_REG_SIZE+8*2(%rsp), %rcx
220 /* regs go into 4th parameter */
223 GLOBAL(ftrace_regs_call)
226 /* Copy flags back to SS, to restore them */
227 movq EFLAGS(%rsp), %rax
228 movq %rax, MCOUNT_REG_SIZE(%rsp)
230 /* Handlers can change the RIP */
232 movq %rax, MCOUNT_REG_SIZE+8(%rsp)
234 /* restore the rest of pt_regs */
248 * As this jmp to ftrace_epilogue can be a short jump
249 * it must not be copied into the trampoline.
250 * The trampoline will add the code to jump
253 GLOBAL(ftrace_regs_caller_end)
257 END(ftrace_regs_caller)
260 #else /* ! CONFIG_DYNAMIC_FTRACE */
263 cmpq $ftrace_stub, ftrace_trace_function
267 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
268 cmpq $ftrace_stub, ftrace_graph_return
269 jnz ftrace_graph_caller
271 cmpq $ftrace_graph_entry_stub, ftrace_graph_entry
272 jnz ftrace_graph_caller
279 /* save_mcount_regs fills in first two parameters */
283 * When DYNAMIC_FTRACE is not defined, ARCH_SUPPORTS_FTRACE_OPS is not
284 * set (see include/asm/ftrace.h and include/linux/ftrace.h). Only the
285 * ip and parent ip are used and the list function is called when
286 * function tracing is enabled.
288 call *ftrace_trace_function
294 #endif /* CONFIG_DYNAMIC_FTRACE */
296 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
297 ENTRY(ftrace_graph_caller)
298 /* Saves rbp into %rdx and fills first parameter */
301 #ifdef CC_USING_FENTRY
302 leaq MCOUNT_REG_SIZE+8(%rsp), %rsi
303 movq $0, %rdx /* No framepointers needed */
305 /* Save address of the return address of traced function */
307 /* ftrace does sanity checks against frame pointers */
310 call prepare_ftrace_return
315 END(ftrace_graph_caller)
317 GLOBAL(return_to_handler)
320 /* Save the return values */
325 call ftrace_return_to_handler