2 * patch.c - livepatch patching functions
4 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
5 * Copyright (C) 2014 SUSE
6 * Copyright (C) 2015 Josh Poimboeuf <jpoimboe@redhat.com>
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version 2
11 * of the License, or (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, see <http://www.gnu.org/licenses/>.
22 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24 #include <linux/livepatch.h>
25 #include <linux/list.h>
26 #include <linux/ftrace.h>
27 #include <linux/rculist.h>
28 #include <linux/slab.h>
29 #include <linux/bug.h>
30 #include <linux/printk.h>
32 #include "transition.h"
34 static LIST_HEAD(klp_ops);
36 struct klp_ops *klp_find_ops(unsigned long old_addr)
39 struct klp_func *func;
41 list_for_each_entry(ops, &klp_ops, node) {
42 func = list_first_entry(&ops->func_stack, struct klp_func,
44 if (func->old_addr == old_addr)
51 static void notrace klp_ftrace_handler(unsigned long ip,
52 unsigned long parent_ip,
53 struct ftrace_ops *fops,
57 struct klp_func *func;
60 ops = container_of(fops, struct klp_ops, fops);
64 func = list_first_or_null_rcu(&ops->func_stack, struct klp_func,
68 * func should never be NULL because preemption should be disabled here
69 * and unregister_ftrace_function() does the equivalent of a
70 * synchronize_sched() before the func_stack removal.
72 if (WARN_ON_ONCE(!func))
76 * In the enable path, enforce the order of the ops->func_stack and
77 * func->transition reads. The corresponding write barrier is in
78 * __klp_enable_patch().
80 * (Note that this barrier technically isn't needed in the disable
81 * path. In the rare case where klp_update_patch_state() runs before
82 * this handler, its TIF_PATCH_PENDING read and this func->transition
83 * read need to be ordered. But klp_update_patch_state() already
88 if (unlikely(func->transition)) {
91 * Enforce the order of the func->transition and
92 * current->patch_state reads. Otherwise we could read an
93 * out-of-date task state and pick the wrong function. The
94 * corresponding write barrier is in klp_init_transition().
98 patch_state = current->patch_state;
100 WARN_ON_ONCE(patch_state == KLP_UNDEFINED);
102 if (patch_state == KLP_UNPATCHED) {
104 * Use the previously patched version of the function.
105 * If no previous patches exist, continue with the
108 func = list_entry_rcu(func->stack_node.next,
109 struct klp_func, stack_node);
111 if (&func->stack_node == &ops->func_stack)
116 klp_arch_set_pc(regs, (unsigned long)func->new_func);
122 * Convert a function address into the appropriate ftrace location.
124 * Usually this is just the address of the function, but on some architectures
125 * it's more complicated so allow them to provide a custom behaviour.
127 #ifndef klp_get_ftrace_location
128 static unsigned long klp_get_ftrace_location(unsigned long faddr)
134 static void klp_unpatch_func(struct klp_func *func)
138 if (WARN_ON(!func->patched))
140 if (WARN_ON(!func->old_addr))
143 ops = klp_find_ops(func->old_addr);
147 if (list_is_singular(&ops->func_stack)) {
148 unsigned long ftrace_loc;
150 ftrace_loc = klp_get_ftrace_location(func->old_addr);
151 if (WARN_ON(!ftrace_loc))
154 WARN_ON(unregister_ftrace_function(&ops->fops));
155 WARN_ON(ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0));
157 list_del_rcu(&func->stack_node);
158 list_del(&ops->node);
161 list_del_rcu(&func->stack_node);
164 func->patched = false;
167 static int klp_patch_func(struct klp_func *func)
172 if (WARN_ON(!func->old_addr))
175 if (WARN_ON(func->patched))
178 ops = klp_find_ops(func->old_addr);
180 unsigned long ftrace_loc;
182 ftrace_loc = klp_get_ftrace_location(func->old_addr);
184 pr_err("failed to find location for function '%s'\n",
189 ops = kzalloc(sizeof(*ops), GFP_KERNEL);
193 ops->fops.func = klp_ftrace_handler;
194 ops->fops.flags = FTRACE_OPS_FL_SAVE_REGS |
195 FTRACE_OPS_FL_DYNAMIC |
196 FTRACE_OPS_FL_IPMODIFY;
198 list_add(&ops->node, &klp_ops);
200 INIT_LIST_HEAD(&ops->func_stack);
201 list_add_rcu(&func->stack_node, &ops->func_stack);
203 ret = ftrace_set_filter_ip(&ops->fops, ftrace_loc, 0, 0);
205 pr_err("failed to set ftrace filter for function '%s' (%d)\n",
206 func->old_name, ret);
210 ret = register_ftrace_function(&ops->fops);
212 pr_err("failed to register ftrace handler for function '%s' (%d)\n",
213 func->old_name, ret);
214 ftrace_set_filter_ip(&ops->fops, ftrace_loc, 1, 0);
220 list_add_rcu(&func->stack_node, &ops->func_stack);
223 func->patched = true;
228 list_del_rcu(&func->stack_node);
229 list_del(&ops->node);
234 void klp_unpatch_object(struct klp_object *obj)
236 struct klp_func *func;
238 klp_for_each_func(obj, func)
240 klp_unpatch_func(func);
242 obj->patched = false;
245 int klp_patch_object(struct klp_object *obj)
247 struct klp_func *func;
250 if (WARN_ON(obj->patched))
253 klp_for_each_func(obj, func) {
254 ret = klp_patch_func(func);
256 klp_unpatch_object(obj);
265 void klp_unpatch_objects(struct klp_patch *patch)
267 struct klp_object *obj;
269 klp_for_each_object(patch, obj)
271 klp_unpatch_object(obj);