diff options
| -rw-r--r-- | include/linux/jump_label.h | 25 | ||||
| -rw-r--r-- | kernel/jump_label.c | 25 |
2 files changed, 21 insertions, 29 deletions
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 6a8b4fe10ad8..0ddb208b8449 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -101,24 +101,9 @@ static inline int static_key_count(struct static_key *key) #ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL -#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL -#define JUMP_LABEL_TYPE_MASK 1UL - -static -inline struct jump_entry *jump_label_get_entries(struct static_key *key) -{ - return (struct jump_entry *)((unsigned long)key->entries - & ~JUMP_LABEL_TYPE_MASK); -} - -static inline bool jump_label_get_branch_default(struct static_key *key) -{ - if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) == - JUMP_LABEL_TYPE_TRUE_BRANCH) - return true; - return false; -} +#define JUMP_TYPE_FALSE 0UL +#define JUMP_TYPE_TRUE 1UL +#define JUMP_TYPE_MASK 1UL static __always_inline bool static_key_false(struct static_key *key) { @@ -147,10 +132,10 @@ extern void jump_label_apply_nops(struct module *mod); #define STATIC_KEY_INIT_TRUE ((struct static_key) \ { .enabled = ATOMIC_INIT(1), \ - .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) + .entries = (void *)JUMP_TYPE_TRUE }) #define STATIC_KEY_INIT_FALSE ((struct static_key) \ { .enabled = ATOMIC_INIT(0), \ - .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH }) + .entries = (void *)JUMP_TYPE_FALSE }) #else /* !HAVE_JUMP_LABEL */ diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 96d8945c8bf3..85a2a0086c67 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -56,6 +56,11 @@ jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop) static void jump_label_update(struct static_key *key, int enable); +static inline bool static_key_type(struct static_key *key) +{ + return (unsigned long)key->entries & JUMP_TYPE_MASK; +} + void static_key_slow_inc(struct static_key *key) { STATIC_KEY_CHECK_USE(); @@ -64,7 +69,7 @@ void static_key_slow_inc(struct static_key *key) jump_label_lock(); if (atomic_read(&key->enabled) == 0) { - if (!jump_label_get_branch_default(key)) + if (!static_key_type(key)) jump_label_update(key, JUMP_LABEL_JMP); else jump_label_update(key, JUMP_LABEL_NOP); @@ -87,7 +92,7 @@ static void __static_key_slow_dec(struct static_key *key, atomic_inc(&key->enabled); schedule_delayed_work(work, rate_limit); } else { - if (!jump_label_get_branch_default(key)) + if (!static_key_type(key)) jump_label_update(key, JUMP_LABEL_NOP); else jump_label_update(key, JUMP_LABEL_JMP); @@ -178,15 +183,17 @@ static void __jump_label_update(struct static_key *key, } } -static enum jump_label_type jump_label_type(struct static_key *key) +static inline struct jump_entry *static_key_entries(struct static_key *key) { - bool true_branch = jump_label_get_branch_default(key); - bool state = static_key_enabled(key); + return (struct jump_entry *)((unsigned long)key->entries & ~JUMP_TYPE_MASK); +} - if ((!true_branch && state) || (true_branch && !state)) - return JUMP_LABEL_JMP; +static enum jump_label_type jump_label_type(struct static_key *key) +{ + bool enabled = static_key_enabled(key); + bool type = static_key_type(key); - return JUMP_LABEL_NOP; + return enabled ^ type; } void __init jump_label_init(void) @@ -442,7 +449,7 @@ int jump_label_text_reserved(void *start, void *end) static void jump_label_update(struct static_key *key, int enable) { struct jump_entry *stop = __stop___jump_table; - struct jump_entry *entry = jump_label_get_entries(key); + struct jump_entry *entry = static_key_entries(key); #ifdef CONFIG_MODULES struct module *mod; |