1 #ifndef _LINUX_JUMP_LABEL_H
2 #define _LINUX_JUMP_LABEL_H
7 * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
8 * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
12 * The use of 'struct static_key' directly, is now DEPRECATED. In addition
13 * static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following:
15 * struct static_key false = STATIC_KEY_INIT_FALSE;
16 * struct static_key true = STATIC_KEY_INIT_TRUE;
20 * The updated API replacements are:
22 * DEFINE_STATIC_KEY_TRUE(key);
23 * DEFINE_STATIC_KEY_FALSE(key);
24 * DEFINE_STATIC_KEY_ARRAY_TRUE(keys, count);
25 * DEFINE_STATIC_KEY_ARRAY_FALSE(keys, count);
26 * static_branch_likely()
27 * static_branch_unlikely()
29 * Jump labels provide an interface to generate dynamic branches using
30 * self-modifying code. Assuming toolchain and architecture support, if we
31 * define a "key" that is initially false via "DEFINE_STATIC_KEY_FALSE(key)",
32 * an "if (static_branch_unlikely(&key))" statement is an unconditional branch
33 * (which defaults to false - and the true block is placed out of line).
34 * Similarly, we can define an initially true key via
35 * "DEFINE_STATIC_KEY_TRUE(key)", and use it in the same
36 * "if (static_branch_unlikely(&key))", in which case we will generate an
37 * unconditional branch to the out-of-line true branch. Keys that are
38 * initially true or false can be using in both static_branch_unlikely()
39 * and static_branch_likely() statements.
41 * At runtime we can change the branch target by setting the key
42 * to true via a call to static_branch_enable(), or false using
43 * static_branch_disable(). If the direction of the branch is switched by
44 * these calls then we run-time modify the branch target via a
45 * no-op -> jump or jump -> no-op conversion. For example, for an
46 * initially false key that is used in an "if (static_branch_unlikely(&key))"
47 * statement, setting the key to true requires us to patch in a jump
48 * to the out-of-line of true branch.
50 * In addition to static_branch_{enable,disable}, we can also reference count
51 * the key or branch direction via static_branch_{inc,dec}. Thus,
52 * static_branch_inc() can be thought of as a 'make more true' and
53 * static_branch_dec() as a 'make more false'.
55 * Since this relies on modifying code, the branch modifying functions
56 * must be considered absolute slow paths (machine wide synchronization etc.).
57 * OTOH, since the affected branches are unconditional, their runtime overhead
58 * will be absolutely minimal, esp. in the default (off) case where the total
59 * effect is a single NOP of appropriate size. The on case will patch in a jump
60 * to the out-of-line block.
62 * When the control is directly exposed to userspace, it is prudent to delay the
63 * decrement to avoid high frequency code modifications which can (and do)
64 * cause significant performance degradation. Struct static_key_deferred and
65 * static_key_slow_dec_deferred() provide for this.
67 * Lacking toolchain and or architecture support, static keys fall back to a
68 * simple conditional branch.
70 * Additional babbling in: Documentation/static-keys.txt
73 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
74 # define HAVE_JUMP_LABEL
79 #include <linux/types.h>
80 #include <linux/compiler.h>
82 extern bool static_key_initialized;
84 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized, \
85 "%s used before call to jump_label_init", \
88 #ifdef HAVE_JUMP_LABEL
92 /* Set lsb bit to 1 if branch is default true, 0 ot */
93 struct jump_entry *entries;
95 struct static_key_mod *next;
103 #endif /* HAVE_JUMP_LABEL */
104 #endif /* __ASSEMBLY__ */
106 #ifdef HAVE_JUMP_LABEL
107 #include <asm/jump_label.h>
112 enum jump_label_type {
119 #ifdef HAVE_JUMP_LABEL
121 #define JUMP_TYPE_FALSE 0UL
122 #define JUMP_TYPE_TRUE 1UL
123 #define JUMP_TYPE_MASK 1UL
125 static __always_inline bool static_key_false(struct static_key *key)
127 return arch_static_branch(key, false);
130 static __always_inline bool static_key_true(struct static_key *key)
132 return !arch_static_branch(key, true);
135 extern struct jump_entry __start___jump_table[];
136 extern struct jump_entry __stop___jump_table[];
138 extern void jump_label_init(void);
139 extern void jump_label_lock(void);
140 extern void jump_label_unlock(void);
141 extern void arch_jump_label_transform(struct jump_entry *entry,
142 enum jump_label_type type);
143 extern void arch_jump_label_transform_static(struct jump_entry *entry,
144 enum jump_label_type type);
145 extern int jump_label_text_reserved(void *start, void *end);
146 extern void static_key_slow_inc(struct static_key *key);
147 extern void static_key_slow_dec(struct static_key *key);
148 extern void jump_label_apply_nops(struct module *mod);
149 extern int static_key_count(struct static_key *key);
150 extern void static_key_enable(struct static_key *key);
151 extern void static_key_disable(struct static_key *key);
154 * We should be using ATOMIC_INIT() for initializing .enabled, but
155 * the inclusion of atomic.h is problematic for inclusion of jump_label.h
156 * in 'low-level' headers. Thus, we are initializing .enabled with a
157 * raw value, but have added a BUILD_BUG_ON() to catch any issues in
158 * jump_label_init() see: kernel/jump_label.c.
160 #define STATIC_KEY_INIT_TRUE \
161 { .enabled = { 1 }, \
162 .entries = (void *)JUMP_TYPE_TRUE }
163 #define STATIC_KEY_INIT_FALSE \
164 { .enabled = { 0 }, \
165 .entries = (void *)JUMP_TYPE_FALSE }
167 #else /* !HAVE_JUMP_LABEL */
169 #include <linux/atomic.h>
170 #include <linux/bug.h>
172 static inline int static_key_count(struct static_key *key)
174 return atomic_read(&key->enabled);
177 static __always_inline void jump_label_init(void)
179 static_key_initialized = true;
182 static __always_inline bool static_key_false(struct static_key *key)
184 if (unlikely(static_key_count(key) > 0))
189 static __always_inline bool static_key_true(struct static_key *key)
191 if (likely(static_key_count(key) > 0))
196 static inline void static_key_slow_inc(struct static_key *key)
198 STATIC_KEY_CHECK_USE();
199 atomic_inc(&key->enabled);
202 static inline void static_key_slow_dec(struct static_key *key)
204 STATIC_KEY_CHECK_USE();
205 atomic_dec(&key->enabled);
208 static inline int jump_label_text_reserved(void *start, void *end)
213 static inline void jump_label_lock(void) {}
214 static inline void jump_label_unlock(void) {}
216 static inline int jump_label_apply_nops(struct module *mod)
221 static inline void static_key_enable(struct static_key *key)
223 int count = static_key_count(key);
225 WARN_ON_ONCE(count < 0 || count > 1);
228 static_key_slow_inc(key);
231 static inline void static_key_disable(struct static_key *key)
233 int count = static_key_count(key);
235 WARN_ON_ONCE(count < 0 || count > 1);
238 static_key_slow_dec(key);
241 #define STATIC_KEY_INIT_TRUE { .enabled = ATOMIC_INIT(1) }
242 #define STATIC_KEY_INIT_FALSE { .enabled = ATOMIC_INIT(0) }
244 #endif /* HAVE_JUMP_LABEL */
246 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
247 #define jump_label_enabled static_key_enabled
249 /* -------------------------------------------------------------------------- */
252 * Two type wrappers around static_key, such that we can use compile time
253 * type differentiation to emit the right code.
255 * All the below code is macros in order to play type games.
258 struct static_key_true {
259 struct static_key key;
262 struct static_key_false {
263 struct static_key key;
266 #define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
267 #define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
269 #define DEFINE_STATIC_KEY_TRUE(name) \
270 struct static_key_true name = STATIC_KEY_TRUE_INIT
272 #define DECLARE_STATIC_KEY_TRUE(name) \
273 extern struct static_key_true name
275 #define DEFINE_STATIC_KEY_FALSE(name) \
276 struct static_key_false name = STATIC_KEY_FALSE_INIT
278 #define DECLARE_STATIC_KEY_FALSE(name) \
279 extern struct static_key_false name
281 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
282 struct static_key_true name[count] = { \
283 [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
286 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
287 struct static_key_false name[count] = { \
288 [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
291 extern bool ____wrong_branch_error(void);
293 #define static_key_enabled(x) \
295 if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
296 !__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
297 !__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
298 ____wrong_branch_error(); \
299 static_key_count((struct static_key *)x) > 0; \
302 #ifdef HAVE_JUMP_LABEL
305 * Combine the right initial value (type) with the right branch order
306 * to generate the desired result.
309 * type\branch| likely (1) | unlikely (0)
310 * -----------+-----------------------+------------------
312 * true (1) | ... | ...
314 * | <br-stmts> | 1: ...
320 * -----------+-----------------------+------------------
322 * false (0) | ... | ...
324 * | <br-stmts> | 1: ...
330 * -----------+-----------------------+------------------
332 * The initial value is encoded in the LSB of static_key::entries,
333 * type: 0 = false, 1 = true.
335 * The branch type is encoded in the LSB of jump_entry::key,
336 * branch: 0 = unlikely, 1 = likely.
338 * This gives the following logic table:
340 * enabled type branch instuction
341 * -----------------------------+-----------
352 * Which gives the following functions:
354 * dynamic: instruction = enabled ^ branch
355 * static: instruction = type ^ branch
357 * See jump_label_type() / jump_label_init_type().
360 #define static_branch_likely(x) \
363 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
364 branch = !arch_static_branch(&(x)->key, true); \
365 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
366 branch = !arch_static_branch_jump(&(x)->key, true); \
368 branch = ____wrong_branch_error(); \
372 #define static_branch_unlikely(x) \
375 if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
376 branch = arch_static_branch_jump(&(x)->key, false); \
377 else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
378 branch = arch_static_branch(&(x)->key, false); \
380 branch = ____wrong_branch_error(); \
384 #else /* !HAVE_JUMP_LABEL */
386 #define static_branch_likely(x) likely(static_key_enabled(&(x)->key))
387 #define static_branch_unlikely(x) unlikely(static_key_enabled(&(x)->key))
389 #endif /* HAVE_JUMP_LABEL */
392 * Advanced usage; refcount, branch is enabled when: count != 0
395 #define static_branch_inc(x) static_key_slow_inc(&(x)->key)
396 #define static_branch_dec(x) static_key_slow_dec(&(x)->key)
399 * Normal usage; boolean enable/disable.
402 #define static_branch_enable(x) static_key_enable(&(x)->key)
403 #define static_branch_disable(x) static_key_disable(&(x)->key)
405 #endif /* _LINUX_JUMP_LABEL_H */
407 #endif /* __ASSEMBLY__ */