ftrace: Use breakpoint method to update ftrace caller

On boot up and module load, it is fine to modify the code directly,
without the use of breakpoints. This is because boot up modification
is done before SMP is initialized, thus the modification is serial,
and module load is done before the module executes.

But after that we must use a SMP safe method to modify running code.
Otherwise, if we are running the function tracer and update its
function (by starting off the stack tracer, or perf tracing)
the change of the function called by the ftrace trampoline is done
directly. If this is being executed on another CPU, that CPU may
take a GPF and crash the kernel.

The breakpoint method is used to change the nops at all the functions, but
the change of the ftrace callback handler itself was still using a
direct modification. If tracing was enabled and the function callback
was changed then another CPU could fault if it was currently calling
the original callback. This modification must use the breakpoint method
too.

Note, the direct method is still used for boot up and module load.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 2407a6d81cb7de3a8cb23ea718b8b4f979cddbf0..c3a7cb4bf6e6f0f429495d8f4a9478ac9161c0d6 100644 (file)
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -100,7 +100,7 @@ static const unsigned char *ftrace_nop_replace(void)
 }
 
 static int
-ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
+ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
                   unsigned const char *new_code)
 {
        unsigned char replaced[MCOUNT_INSN_SIZE];
@@ -141,7 +141,20 @@ int ftrace_make_nop(struct module *mod,
        old = ftrace_call_replace(ip, addr);
        new = ftrace_nop_replace();
 
-       return ftrace_modify_code(rec->ip, old, new);
+       /*
+        * On boot up, and when modules are loaded, the MCOUNT_ADDR
+        * is converted to a nop, and will never become MCOUNT_ADDR
+        * again. This code is either running before SMP (on boot up)
+        * or before the code will ever be executed (module load).
+        * We do not want to use the breakpoint version in this case,
+        * just modify the code directly.
+        */
+       if (addr == MCOUNT_ADDR)
+               return ftrace_modify_code_direct(rec->ip, old, new);
+
+       /* Normal cases use add_brk_on_nop */
+       WARN_ONCE(1, "invalid use of ftrace_make_nop");
+       return -EINVAL;
 }
 
 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
@@ -152,20 +165,8 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
        old = ftrace_nop_replace();
        new = ftrace_call_replace(ip, addr);
 
-       return ftrace_modify_code(rec->ip, old, new);
-}
-
-int ftrace_update_ftrace_func(ftrace_func_t func)
-{
-       unsigned long ip = (unsigned long)(&ftrace_call);
-       unsigned char old[MCOUNT_INSN_SIZE], *new;
-       int ret;
-
-       memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
-       new = ftrace_call_replace(ip, (unsigned long)func);
-       ret = ftrace_modify_code(ip, old, new);
-
-       return ret;
+       /* Should only be called when module is loaded */
+       return ftrace_modify_code_direct(rec->ip, old, new);
 }
 
 /*
@@ -201,6 +202,29 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
  */
 atomic_t modifying_ftrace_code __read_mostly;
 
+static int
+ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
+                  unsigned const char *new_code);
+
+int ftrace_update_ftrace_func(ftrace_func_t func)
+{
+       unsigned long ip = (unsigned long)(&ftrace_call);
+       unsigned char old[MCOUNT_INSN_SIZE], *new;
+       int ret;
+
+       memcpy(old, &ftrace_call, MCOUNT_INSN_SIZE);
+       new = ftrace_call_replace(ip, (unsigned long)func);
+
+       /* See comment above by declaration of modifying_ftrace_code */
+       atomic_inc(&modifying_ftrace_code);
+
+       ret = ftrace_modify_code(ip, old, new);
+
+       atomic_dec(&modifying_ftrace_code);
+
+       return ret;
+}
+
 /*
  * A breakpoint was added to the code address we are about to
  * modify, and this is the handle that will just skip over it.
@@ -520,6 +544,38 @@ void ftrace_replace_code(int enable)
        }
 }
 
+static int
+ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
+                  unsigned const char *new_code)
+{
+       int ret;
+
+       ret = add_break(ip, old_code);
+       if (ret)
+               goto out;
+
+       run_sync();
+
+       ret = add_update_code(ip, new_code);
+       if (ret)
+               goto fail_update;
+
+       run_sync();
+
+       ret = ftrace_write(ip, new_code, 1);
+       if (ret) {
+               ret = -EPERM;
+               goto out;
+       }
+       run_sync();
+ out:
+       return ret;
+
+ fail_update:
+       probe_kernel_write((void *)ip, &old_code[0], 1);
+       goto out;
+}
+
 void arch_ftrace_update_code(int command)
 {
        /* See comment above by declaration of modifying_ftrace_code */