jump_label: Another (better) static_key interface
Subsystems (9)
Peter Zijlstra
maintainercover jump_label: Another (better) static_key interface
Hi all,
After yet another bug because of the weirdness of the static key interface, here an attempt at providing a better one.
This is boot tested on x86_64, bzImage works, modules have a problem. It looks like jump_label_add_module() runs into a 'wrong' NOP, which would indicate jump_label_apply_nops() didn't work right.
I'm too tired to spot the fail, so I figured I'd post it anyway :-)
Beware: x86_64_defconfig has CONFIG_JUMP_LABEL=n.
@arch people, please have a look at patch 6 where I've attempted to do inline asm for all kinds of unknown archs.
--- arch/arm/include/asm/jump_label.h | 18 +++- arch/arm/kernel/jump_label.c | 2 +- arch/arm64/include/asm/jump_label.h | 18 +++- arch/arm64/kernel/jump_label.c | 2 +- arch/mips/include/asm/jump_label.h | 19 +++- arch/mips/kernel/jump_label.c | 2 +- arch/powerpc/include/asm/jump_label.h | 19 +++- arch/powerpc/kernel/jump_label.c | 2 +- arch/s390/include/asm/jump_label.h | 19 +++- arch/s390/kernel/jump_label.c | 2 +- arch/sparc/include/asm/jump_label.h | 34 ++++-- arch/sparc/kernel/jump_label.c | 2 +- arch/x86/include/asm/jump_label.h | 21 +++- arch/x86/kernel/jump_label.c | 2 +- arch/x86/kernel/tsc.c | 22 ++-- include/linux/jump_label.h | 188 +++++++++++++++++++++++++++++----- kernel/jump_label.c | 114 ++++++++++++--------- kernel/sched/core.c | 6 +- 18 files changed, 374 insertions(+), 118 deletions(-)
Peter Zijlstra
maintainerRFC PATCH 1/7 jump_label: Rename JUMP_LABEL_{EN,DIS}ABLE
Since we've already stepped away from ENABLE is a JMP and DISABLE is a NOP with the branch_default bits, and are going to make it even worse, rename to make it clearer.
--- arch/arm/kernel/jump_label.c | 2 +- arch/arm64/kernel/jump_label.c | 2 +- arch/mips/kernel/jump_label.c | 2 +- arch/powerpc/kernel/jump_label.c | 2 +- arch/s390/kernel/jump_label.c | 2 +- arch/sparc/kernel/jump_label.c | 2 +- arch/x86/kernel/jump_label.c | 2 +- include/linux/jump_label.h | 4 ++-- kernel/jump_label.c | 18 +++++++++--------- 9 files changed, 18 insertions(+), 18 deletions(-)
arch/arm/kernel/jump_label.c +1 -1
@@ -12,7 +12,7 @@ static void __arch_jump_label_transform(void *addr = (void *)entry->code;unsigned int insn; -if (type ==JUMP_LABEL_ENABLE ) +if (type ==JUMP_LABEL_JMP ) insn =arm_gen_branch (entry->code, entry->target);else insn =arm_gen_nop ();
arch/arm64/kernel/jump_label.c +1 -1
@@ -28,7 +28,7 @@ void arch_jump_label_transform(struct juvoid *addr = (void *)entry->code;u32 insn; -if (type ==JUMP_LABEL_ENABLE ) { +if (type ==JUMP_LABEL_JMP ) { insn =aarch64_insn_gen_branch_imm (entry->code, entry->target,AARCH64_INSN_BRANCH_NOLINK );
arch/mips/kernel/jump_label.c +1 -1
@@ -51,7 +51,7 @@ void arch_jump_label_transform(struct ju/* Target must have the right alignment and ISA must be preserved. */ BUG_ON ((e->target &J_ALIGN_MASK ) !=J_ISA_BIT ); -if (type ==JUMP_LABEL_ENABLE ) { +if (type ==JUMP_LABEL_JMP ) { insn.j_format.opcode =J_ISA_BIT ? mm_j32_op : j_op; insn.j_format.target = e->target >>J_RANGE_SHIFT ; }else {
arch/powerpc/kernel/jump_label.c +1 -1
@@ -17,7 +17,7 @@ void arch_jump_label_transform(struct ju {u32 *addr = (u32 *)(unsigned long )entry->code; -if (type ==JUMP_LABEL_ENABLE ) +if (type ==JUMP_LABEL_JMP )patch_branch (addr, entry->target,0 );else patch_instruction (addr,PPC_INST_NOP );
arch/s390/kernel/jump_label.c +1 -1
@@ -64,7 +64,7 @@ static void __jump_label_transform(struc {struct insn old, new; -if (type ==JUMP_LABEL_ENABLE ) { +if (type ==JUMP_LABEL_JMP ) {jump_label_make_nop (entry, &old);jump_label_make_branch (entry, &new); }else {
arch/sparc/kernel/jump_label.c +1 -1
@@ -16,7 +16,7 @@ void arch_jump_label_transform(struct juu32 val;u32 *insn = (u32 *) (unsigned long ) entry->code; -if (type ==JUMP_LABEL_ENABLE ) { +if (type ==JUMP_LABEL_JMP ) {s32 off = (s32 )entry->target - (s32 )entry->code;#ifdef CONFIG_SPARC64
arch/x86/kernel/jump_label.c +1 -1
@@ -45,7 +45,7 @@ static void __jump_label_transform(strucconst unsigned char default_nop[] = {STATIC_KEY_INIT_NOP };const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5 ]; -if (type ==JUMP_LABEL_ENABLE ) { +if (type ==JUMP_LABEL_JMP ) {if (init) {/* * Jump label is enabled for the first time.
include/linux/jump_label.h +2 -2
@@ -86,8 +86,8 @@ struct static_key {#ifndef __ASSEMBLY__enum jump_label_type { -JUMP_LABEL_DISABLE =0 , -JUMP_LABEL_ENABLE , +JUMP_LABEL_NOP =0 , +JUMP_LABEL_JMP , };struct module ;
kernel/jump_label.c +9 -9
@@ -65,9 +65,9 @@ void static_key_slow_inc(struct static_kjump_label_lock ();if (atomic_read (&key->enabled) ==0 ) {if (!jump_label_get_branch_default (key)) -jump_label_update (key,JUMP_LABEL_ENABLE ); +jump_label_update (key,JUMP_LABEL_JMP );else -jump_label_update (key,JUMP_LABEL_DISABLE ); +jump_label_update (key,JUMP_LABEL_NOP ); }atomic_inc (&key->enabled);jump_label_unlock ();
@@ -88,9 +88,9 @@ static void __static_key_slow_dec(structschedule_delayed_work (work, rate_limit); }else {if (!jump_label_get_branch_default (key)) -jump_label_update (key,JUMP_LABEL_DISABLE ); +jump_label_update (key,JUMP_LABEL_NOP );else -jump_label_update (key,JUMP_LABEL_ENABLE ); +jump_label_update (key,JUMP_LABEL_JMP ); }jump_label_unlock (); }
@@ -184,9 +184,9 @@ static enum jump_label_type jump_label_tbool state =static_key_enabled (key);if ((!true_branch && state) || (true_branch && !state)) -return JUMP_LABEL_ENABLE ; +return JUMP_LABEL_JMP ; -return JUMP_LABEL_DISABLE ; +return JUMP_LABEL_NOP ; }void __init jump_label_init (void )
@@ -276,7 +276,7 @@ void jump_label_apply_nops(struct modulereturn ;for (iter = iter_start; iter < iter_stop; iter++) { -arch_jump_label_transform_static (iter,JUMP_LABEL_DISABLE ); +arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); } }
@@ -318,8 +318,8 @@ static int jump_label_add_module(struct jlm->next = key->next; key->next = jlm; -if (jump_label_type (key) ==JUMP_LABEL_ENABLE ) -__jump_label_update (key, iter, iter_stop,JUMP_LABEL_ENABLE ); +if (jump_label_type (key) ==JUMP_LABEL_JMP ) +__jump_label_update (key, iter, iter_stop,JUMP_LABEL_JMP ); }return 0 ;
Peter Zijlstra
maintainerRFC PATCH 2/7 jump_label: Rename JUMP_LABEL_TYPE_*
In reply to Peter ZijlstraRename the JUMP_LABEL_TYPE_* macros to be JUMP_TYPE_* and move the inline helpers into kernel/jump_label.c, since that's the only place they're ever used.
Also rename the helpers.
--- include/linux/jump_label.h | 25 +++++-------------------- kernel/jump_label.c | 25 ++++++++++++++++--------- 2 files changed, 21 insertions(+), 29 deletions(-)
include/linux/jump_label.h +5 -20
@@ -101,24 +101,9 @@ static inline int static_key_count(struc#ifdef HAVE_JUMP_LABEL -#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL -#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL -#define JUMP_LABEL_TYPE_MASK 1UL - -static -inline struct jump_entry *jump_label_get_entries (struct static_key *key) -{ -return (struct jump_entry *)((unsigned long )key->entries - & ~JUMP_LABEL_TYPE_MASK ); -} - -static inline bool jump_label_get_branch_default (struct static_key *key) -{ -if (((unsigned long )key->entries &JUMP_LABEL_TYPE_MASK ) == -JUMP_LABEL_TYPE_TRUE_BRANCH ) -return true; -return false; -} +#define JUMP_TYPE_FALSE 0UL +#define JUMP_TYPE_TRUE 1UL +#define JUMP_TYPE_MASK 1UL static __always_inline bool static_key_false(struct static_key *key) {
@@ -147,10 +132,10 @@ extern void jump_label_apply_nops(struct#define STATIC_KEY_INIT_TRUE ((struct static_key) \ { .enabled = ATOMIC_INIT(1), \ - .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH }) + .entries = (void *)JUMP_TYPE_TRUE })#define STATIC_KEY_INIT_FALSE ((struct static_key) \ { .enabled = ATOMIC_INIT(0), \ - .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH }) + .entries = (void *)JUMP_TYPE_FALSE })#else /* !HAVE_JUMP_LABEL */
kernel/jump_label.c +16 -9
@@ -56,6 +56,11 @@ jump_label_sort_entries(struct jump_entrstatic void jump_label_update (struct static_key *key,int enable); +static inline bool static_key_type (struct static_key *key) +{ +return (unsigned long )key->entries &JUMP_TYPE_MASK ; +} +void static_key_slow_inc (struct static_key *key) {STATIC_KEY_CHECK_USE ();
@@ -64,7 +69,7 @@ void static_key_slow_inc(struct static_kjump_label_lock ();if (atomic_read (&key->enabled) ==0 ) { -if (!jump_label_get_branch_default (key)) +if (!static_key_type (key))jump_label_update (key,JUMP_LABEL_JMP );else jump_label_update (key,JUMP_LABEL_NOP );
@@ -87,7 +92,7 @@ static void __static_key_slow_dec(structatomic_inc (&key->enabled);schedule_delayed_work (work, rate_limit); }else { -if (!jump_label_get_branch_default (key)) +if (!static_key_type (key))jump_label_update (key,JUMP_LABEL_NOP );else jump_label_update (key,JUMP_LABEL_JMP );
@@ -178,15 +183,17 @@ static void __jump_label_update(struct s } } -static enum jump_label_type jump_label_type (struct static_key *key) +static inline struct jump_entry *static_key_entries (struct static_key *key) { -bool true_branch =jump_label_get_branch_default (key); -bool state =static_key_enabled (key); +return (struct jump_entry *)((unsigned long )key->entries & ~JUMP_TYPE_MASK ); +} -if ((!true_branch && state) || (true_branch && !state)) -return JUMP_LABEL_JMP ; +static enum jump_label_type jump_label_type (struct static_key *key) +{ +bool enabled =static_key_enabled (key); +bool type =static_key_type (key); -return JUMP_LABEL_NOP ; +return enabled ^ type; }void __init jump_label_init (void )
@@ -442,7 +449,7 @@ int jump_label_text_reserved(void *startstatic void jump_label_update (struct static_key *key,int enable) {struct jump_entry *stop = __stop___jump_table; -struct jump_entry *entry =jump_label_get_entries (key); +struct jump_entry *entry =static_key_entries (key);#ifdef CONFIG_MODULES struct module *mod;
Peter Zijlstra
maintainerRFC PATCH 3/7 jump_label: Add jump_entry_key() helper
In reply to Peter ZijlstraAvoid some casting with a helper, also prepares the way for overloading the LSB of jump_entry::key.
--- kernel/jump_label.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-)
kernel/jump_label.c +10 -5
@@ -188,6 +188,11 @@ static inline struct jump_entry *static_return (struct jump_entry *)((unsigned long )key->entries & ~JUMP_TYPE_MASK ); } +static inline struct static_key *jump_entry_key (struct jump_entry *entry) +{ +return (struct static_key *)((unsigned long )entry->key); +} +static enum jump_label_type jump_label_type (struct static_key *key) {bool enabled =static_key_enabled (key);
@@ -209,8 +214,8 @@ void __init jump_label_init(void)for (iter = iter_start; iter < iter_stop; iter++) {struct static_key *iterk; - iterk = (struct static_key *)(unsigned long )iter->key; -arch_jump_label_transform_static (iter,jump_label_type (iterk)); + iterk =jump_entry_key (iter); +arch_jump_label_transform_static (iter,jump_label_type (iterk));if (iterk == key)continue ;
@@ -304,7 +309,7 @@ static int jump_label_add_module(structfor (iter = iter_start; iter < iter_stop; iter++) {struct static_key *iterk; - iterk = (struct static_key *)(unsigned long )iter->key; + iterk =jump_entry_key (iter);if (iterk == key)continue ;
@@ -341,10 +346,10 @@ static void jump_label_del_module(structstruct static_key_mod *jlm, **prev;for (iter = iter_start; iter < iter_stop; iter++) { -if (iter->key == (jump_label_t )(unsigned long )key) +if (jump_entry_key (iter) == key)continue ; - key = (struct static_key *)(unsigned long )iter->key; + key =jump_entry_key (iter);if (within_module (iter->key, mod))continue ;
Peter Zijlstra
maintainerRFC PATCH 4/7 jump_label: Add static_key_{en,dis}able() helpers
In reply to Peter ZijlstraAdd two helpers to make it easier to treat the refcount as boolean.
--- include/linux/jump_label.h | 20 ++++++++++++++++++++ kernel/sched/core.c | 6 ++---- 2 files changed, 22 insertions(+), 4 deletions(-)
include/linux/jump_label.h +20 -0
@@ -198,6 +198,26 @@ static inline bool static_key_enabled(streturn static_key_count (key) >0 ; } +static inline void static_key_enable (struct static_key *key) +{ +int count =static_key_count (key); + +WARN_ON_ONCE (count <0 || count >1 ); + +if (!count) +static_key_slow_inc (key); +} + +static inline void static_key_disable (struct static_key *key) +{ +int count =static_key_count (key); + +WARN_ON_ONCE (count <0 || count >1 ); + +if (count) +static_key_slow_dec (key); +} +#endif /* _LINUX_JUMP_LABEL_H */ #endif /* __ASSEMBLY__ */
kernel/sched/core.c +2 -4
@@ -164,14 +164,12 @@ struct static_key sched_feat_keys[__SCHEstatic void sched_feat_disable (int i) { -if (static_key_enabled (&sched_feat_keys[i])) -static_key_slow_dec (&sched_feat_keys[i]); +static_key_disable (&sched_feat_keys[i]); }static void sched_feat_enable (int i) { -if (!static_key_enabled (&sched_feat_keys[i])) -static_key_slow_inc (&sched_feat_keys[i]); +static_key_enable (&sched_feat_keys[i]); }#else static void sched_feat_disable (int i) { };
Peter Zijlstra
maintainerRFC PATCH 5/7 jump_label: Rework update logic
In reply to Peter ZijlstraInstead of spreading the branch_default logic all over the place, concentrate it into the one jump_label_type() function.
This does mean we need to actually increment/decrement the enabled count _before_ calling the update path, otherwise jump_label_type() will not see the right state.
--- kernel/jump_label.c | 87 ++++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 50 deletions(-)
kernel/jump_label.c +37 -50
@@ -54,12 +54,7 @@ jump_label_sort_entries(struct jump_entrsort (start, size,sizeof (struct jump_entry ), jump_label_cmp,NULL ); } -static void jump_label_update (struct static_key *key,int enable); - -static inline bool static_key_type (struct static_key *key) -{ -return (unsigned long )key->entries &JUMP_TYPE_MASK ; -} +static void jump_label_update (struct static_key *key);void static_key_slow_inc (struct static_key *key) {
@@ -68,13 +63,8 @@ void static_key_slow_inc(struct static_kreturn ;jump_label_lock (); -if (atomic_read (&key->enabled) ==0 ) { -if (!static_key_type (key)) -jump_label_update (key,JUMP_LABEL_JMP ); -else -jump_label_update (key,JUMP_LABEL_NOP ); - } -atomic_inc (&key->enabled); +if (atomic_inc_and_test (&key->enabled)) +jump_label_update (key);jump_label_unlock (); }EXPORT_SYMBOL_GPL (static_key_slow_inc);
@@ -92,10 +82,7 @@ static void __static_key_slow_dec(structatomic_inc (&key->enabled);schedule_delayed_work (work, rate_limit); }else { -if (!static_key_type (key)) -jump_label_update (key,JUMP_LABEL_NOP ); -else -jump_label_update (key,JUMP_LABEL_JMP ); +jump_label_update (key); }jump_label_unlock (); }
@@ -154,7 +141,7 @@ static int __jump_label_text_reserved(streturn 0 ; } -/* +/* * Update code which is definitely not currently executing. * Architectures which need heavyweight synchronization to modify * running code can override this to make the non-live update case
@@ -163,29 +150,17 @@ static int __jump_label_text_reserved(st void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,enum jump_label_type type) { -arch_jump_label_transform (entry, type); +arch_jump_label_transform (entry, type); } -static void __jump_label_update (struct static_key *key, -struct jump_entry *entry, -struct jump_entry *stop,int enable) +static inline struct jump_entry *static_key_entries (struct static_key *key) { -for (; (entry < stop) && - (entry->key == (jump_label_t )(unsigned long )key); - entry++) { -/* -* entry->code set to 0 invalidates module init text sections -* kernel_text_address() verifies we are not in core kernel -* init code, see jump_label_invalidate_module_init(). -*/ -if (entry->code &&kernel_text_address (entry->code)) -arch_jump_label_transform (entry, enable); - } +return (struct jump_entry *)((unsigned long )key->entries & ~JUMP_TYPE_MASK ); } -static inline struct jump_entry *static_key_entries (struct static_key *key) +static inline bool static_key_type (struct static_key *key) { -return (struct jump_entry *)((unsigned long )key->entries & ~JUMP_TYPE_MASK ); +return (unsigned long )key->entries &JUMP_TYPE_MASK ; }static inline struct static_key *jump_entry_key (struct jump_entry *entry)
@@ -193,14 +168,30 @@ static inline struct static_key *jump_enreturn (struct static_key *)((unsigned long )entry->key); } -static enum jump_label_type jump_label_type (struct static_key *key) +static enum jump_label_type jump_label_type (struct jump_entry *entry) { +struct static_key *key =jump_entry_key (entry);bool enabled =static_key_enabled (key);bool type =static_key_type (key);return enabled ^ type; } +static void __jump_label_update (struct static_key *key, +struct jump_entry *entry, +struct jump_entry *stop) +{ +for (; (entry < stop) && (jump_entry_key (entry) == key); entry++) { +/* +* entry->code set to 0 invalidates module init text sections +* kernel_text_address() verifies we are not in core kernel +* init code, see jump_label_invalidate_module_init(). +*/ +if (entry->code &&kernel_text_address (entry->code)) +arch_jump_label_transform (entry,jump_label_type (entry)); + } +} +void __init jump_label_init (void ) {struct jump_entry *iter_start = __start___jump_table;
@@ -214,8 +205,8 @@ void __init jump_label_init(void)for (iter = iter_start; iter < iter_stop; iter++) {struct static_key *iterk; +arch_jump_label_transform_static (iter,jump_label_type (iter)); iterk =jump_entry_key (iter); -arch_jump_label_transform_static (iter,jump_label_type (iterk));if (iterk == key)continue ;
@@ -255,17 +246,15 @@ static int __jump_label_mod_text_reserve start, end); } -static void __jump_label_mod_update (struct static_key *key,int enable) +static void __jump_label_mod_update (struct static_key *key) { -struct static_key_mod *mod = key->next; +struct static_key_mod *mod; -while (mod) { +for (mod = key->next; mod; mod = mod->next) {struct module *m = mod->mod;__jump_label_update (key, mod->entries, - m->jump_entries + m->num_jump_entries, - enable); - mod = mod->next; + m->jump_entries + m->num_jump_entries); } }
@@ -287,9 +276,8 @@ void jump_label_apply_nops(struct moduleif (iter_start == iter_stop)return ; -for (iter = iter_start; iter < iter_stop; iter++) { +for (iter = iter_start; iter < iter_stop; iter++)arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); - } }static int jump_label_add_module (struct module *mod)
@@ -330,8 +318,7 @@ static int jump_label_add_module(struct jlm->next = key->next; key->next = jlm; -if (jump_label_type (key) ==JUMP_LABEL_JMP ) -__jump_label_update (key, iter, iter_stop,JUMP_LABEL_JMP ); +__jump_label_update (key, iter, iter_stop); }return 0 ;
@@ -451,14 +438,14 @@ int jump_label_text_reserved(void *startreturn ret; } -static void jump_label_update (struct static_key *key,int enable) +static void jump_label_update (struct static_key *key) {struct jump_entry *stop = __stop___jump_table;struct jump_entry *entry =static_key_entries (key);#ifdef CONFIG_MODULES struct module *mod; -__jump_label_mod_update (key, enable); +__jump_label_mod_update (key);preempt_disable (); mod =__module_address ((unsigned long )key);
@@ -468,7 +455,7 @@ static void jump_label_update(struct sta#endif /* if there are no users, entry can be NULL */ if (entry) - __jump_label_update(key, entry, stop, enable); + __jump_label_update(key, entry, stop); }#endif
Peter Zijlstra
maintainerRe: [RFC][PATCH 5/7] jump_label: Rework update logic
kernel/jump_label.c static int jump_label_add_module(struct · line 324
Quoted 10 lines · Peter Zijlstra
kernel/jump_label.c +1
-2@@ -330,8 +318,7 @@ static int jump_label_add_module(struct jlm->next = key->next; key->next = jlm; -if (jump_label_type (key) ==JUMP_LABEL_JMP ) -__jump_label_update (key, iter, iter_stop,JUMP_LABEL_JMP ); +__jump_label_update (key, iter, iter_stop); }return 0 ;
That is the fail; the arch_jump_label_transform() function (at least on x86) double checks that we're changing the branch, so we cannot unconditionally write whatever it should be.
Peter Zijlstra
maintainerRe: [RFC][PATCH 5/7] jump_label: Rework update logic
In reply to Peter ZijlstraFixes jump_label_add_module()
---
Subject: jump_label: Rework update logic
From: Peter Zijlstra <[email protected]>
Date: Fri Jul 24 15:06:37 CEST 2015
Instead of spreading the branch_default logic all over the place, concentrate it into the one jump_label_type() function.
This does mean we need to actually increment/decrement they enabled count _before_ calling the update path, otherwise jump_label_type() will not see the right state.
--- kernel/jump_label.c | 88 ++++++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 50 deletions(-)
kernel/jump_label.c +38 -50
@@ -54,12 +54,7 @@ jump_label_sort_entries(struct jump_entrsort (start, size,sizeof (struct jump_entry ), jump_label_cmp,NULL ); } -static void jump_label_update (struct static_key *key,int enable); - -static inline bool static_key_type (struct static_key *key) -{ -return (unsigned long )key->entries &JUMP_TYPE_MASK ; -} +static void jump_label_update (struct static_key *key);void static_key_slow_inc (struct static_key *key) {
@@ -68,13 +63,8 @@ void static_key_slow_inc(struct static_kreturn ;jump_label_lock (); -if (atomic_read (&key->enabled) ==0 ) { -if (!static_key_type (key)) -jump_label_update (key,JUMP_LABEL_JMP ); -else -jump_label_update (key,JUMP_LABEL_NOP ); - } -atomic_inc (&key->enabled); +if (atomic_inc_and_test (&key->enabled)) +jump_label_update (key);jump_label_unlock (); }EXPORT_SYMBOL_GPL (static_key_slow_inc);
@@ -92,10 +82,7 @@ static void __static_key_slow_dec(structatomic_inc (&key->enabled);schedule_delayed_work (work, rate_limit); }else { -if (!static_key_type (key)) -jump_label_update (key,JUMP_LABEL_NOP ); -else -jump_label_update (key,JUMP_LABEL_JMP ); +jump_label_update (key); }jump_label_unlock (); }
@@ -154,7 +141,7 @@ static int __jump_label_text_reserved(streturn 0 ; } -/* +/* * Update code which is definitely not currently executing. * Architectures which need heavyweight synchronization to modify * running code can override this to make the non-live update case
@@ -163,29 +150,17 @@ static int __jump_label_text_reserved(st void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,enum jump_label_type type) { -arch_jump_label_transform (entry, type); +arch_jump_label_transform (entry, type); } -static void __jump_label_update (struct static_key *key, -struct jump_entry *entry, -struct jump_entry *stop,int enable) +static inline struct jump_entry *static_key_entries (struct static_key *key) { -for (; (entry < stop) && - (entry->key == (jump_label_t )(unsigned long )key); - entry++) { -/* -* entry->code set to 0 invalidates module init text sections -* kernel_text_address() verifies we are not in core kernel -* init code, see jump_label_invalidate_module_init(). -*/ -if (entry->code &&kernel_text_address (entry->code)) -arch_jump_label_transform (entry, enable); - } +return (struct jump_entry *)((unsigned long )key->entries & ~JUMP_TYPE_MASK ); } -static inline struct jump_entry *static_key_entries (struct static_key *key) +static inline bool static_key_type (struct static_key *key) { -return (struct jump_entry *)((unsigned long )key->entries & ~JUMP_TYPE_MASK ); +return (unsigned long )key->entries &JUMP_TYPE_MASK ; }static inline struct static_key *jump_entry_key (struct jump_entry *entry)
@@ -193,14 +168,30 @@ static inline struct static_key *jump_enreturn (struct static_key *)((unsigned long )entry->key); } -static enum jump_label_type jump_label_type (struct static_key *key) +static enum jump_label_type jump_label_type (struct jump_entry *entry) { +struct static_key *key =jump_entry_key (entry);bool enabled =static_key_enabled (key);bool type =static_key_type (key);return enabled ^ type; } +static void __jump_label_update (struct static_key *key, +struct jump_entry *entry, +struct jump_entry *stop) +{ +for (; (entry < stop) && (jump_entry_key (entry) == key); entry++) { +/* +* entry->code set to 0 invalidates module init text sections +* kernel_text_address() verifies we are not in core kernel +* init code, see jump_label_invalidate_module_init(). +*/ +if (entry->code &&kernel_text_address (entry->code)) +arch_jump_label_transform (entry,jump_label_type (entry)); + } +} +void __init jump_label_init (void ) {struct jump_entry *iter_start = __start___jump_table;
@@ -214,8 +205,8 @@ void __init jump_label_init(void)for (iter = iter_start; iter < iter_stop; iter++) {struct static_key *iterk; +arch_jump_label_transform_static (iter,jump_label_type (iter)); iterk =jump_entry_key (iter); -arch_jump_label_transform_static (iter,jump_label_type (iterk));if (iterk == key)continue ;
@@ -255,17 +246,15 @@ static int __jump_label_mod_text_reserve start, end); } -static void __jump_label_mod_update (struct static_key *key,int enable) +static void __jump_label_mod_update (struct static_key *key) { -struct static_key_mod *mod = key->next; +struct static_key_mod *mod; -while (mod) { +for (mod = key->next; mod; mod = mod->next) {struct module *m = mod->mod;__jump_label_update (key, mod->entries, - m->jump_entries + m->num_jump_entries, - enable); - mod = mod->next; + m->jump_entries + m->num_jump_entries); } }
@@ -287,9 +276,8 @@ void jump_label_apply_nops(struct moduleif (iter_start == iter_stop)return ; -for (iter = iter_start; iter < iter_stop; iter++) { +for (iter = iter_start; iter < iter_stop; iter++)arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); - } }static int jump_label_add_module (struct module *mod)
@@ -330,8 +318,8 @@ static int jump_label_add_module(struct jlm->next = key->next; key->next = jlm; -if (jump_label_type (key) ==JUMP_LABEL_JMP ) -__jump_label_update (key, iter, iter_stop,JUMP_LABEL_JMP ); +if (jump_label_type (iter) ==JUMP_LABEL_JMP ) +__jump_label_update (key, iter, iter_stop); }return 0 ;
@@ -451,14 +439,14 @@ int jump_label_text_reserved(void *startreturn ret; } -static void jump_label_update (struct static_key *key,int enable) +static void jump_label_update (struct static_key *key) {struct jump_entry *stop = __stop___jump_table;struct jump_entry *entry =static_key_entries (key);#ifdef CONFIG_MODULES struct module *mod; -__jump_label_mod_update (key, enable); +__jump_label_mod_update (key);preempt_disable (); mod =__module_address ((unsigned long )key);
@@ -468,7 +456,7 @@ static void jump_label_update(struct sta#endif /* if there are no users, entry can be NULL */ if (entry) - __jump_label_update(key, entry, stop, enable); + __jump_label_update(key, entry, stop); }#endif
Peter Zijlstra
maintainerRe: [RFC][PATCH 5/7] jump_label: Rework update logic
kernel/jump_label.c void static_key_slow_inc(struct static_k · line 66
Quoted 5 lines · Peter Zijlstra
kernel/jump_label.c +1
-0@@ -68,13 +63,8 @@ void static_key_slow_inc(struct static_kreturn ;jump_label_lock (); +if (atomic_inc_and_test (&key->enabled))
atomic_inc_return() == 1, works _much_ better.
Quoted 3 lines
kernel/jump_label.c +1
-0+jump_label_update (key);jump_label_unlock (); }
Re: [RFC][PATCH 5/7] jump_label: Rework update logic
Quoted 8 lines · Peter Zijlstra
kernel/jump_label.c void static_key_slow_inc(struct static_k · line 66
Quoted 5 lines · Peter Zijlstra
kernel/jump_label.c +1
-0@@ -68,13 +63,8 @@ void static_key_slow_inc(struct static_kreturn ;jump_label_lock (); +if (atomic_inc_and_test (&key->enabled))
atomic_inc_return()== 1, works _much_ better.
Confirmed. With this change, the jump_label_init() patch, plus the changes in s390 inline asm everything works for me.
There are a couple of places in s390 code where the DEFINE_STATIC_KEY_FALSE and static_branch_likely() combination would make sense.
Peter Zijlstra
maintainerRFC PATCH 6/7 jump_label: Add a new static_key interface
In reply to Peter ZijlstraThere are various problems and short-comings with the current static_key interface:
- static_key_{true,false}() read like a branch depending on the key value, instead of the actual likely/unlikely branch depending on init value.
- static_key_{true,false}() are, as stated above, tied to the static_key init values STATIC_KEY_INIT_{TRUE,FALSE}.
- we're limited to the 2 (out of 4) possible options that compile to a default NOP because that's what our arch_static_branch() assembly emits.
So provide a new static_key interface:
DEFINE_STATIC_KEY_TRUE(name); DEFINE_STATIC_KEY_FALSE(name);
Which defines a key of different types with an initial true/false value.
Then allow:
static_branch_likely()
static_branch_unlikely()
to take a key of either type and emit the right instruction for the case.
This means adding a second arch_static_branch_jump() assembly helper which emits a JMP per default.
XXX for archs that have variable width jmp encodings, please veryify
the right sized jump is emitted.
In order to determine the right instruction for the right state, encode the branch type in the LSB of jump_entry::key.
--- arch/arm/include/asm/jump_label.h | 18 +++- arch/arm64/include/asm/jump_label.h | 18 +++- arch/mips/include/asm/jump_label.h | 19 ++++ arch/powerpc/include/asm/jump_label.h | 19 ++++ arch/s390/include/asm/jump_label.h | 19 ++++ arch/sparc/include/asm/jump_label.h | 34 +++++--- arch/x86/include/asm/jump_label.h | 21 ++++ include/linux/jump_label.h | 143 ++++++++++++++++++++++++++++++++-- kernel/jump_label.c | 27 +++++- 9 files changed, 287 insertions(+), 31 deletions(-)
arch/arm/include/asm/jump_label.h +16 -2
@@ -13,14 +13,28 @@#define JUMP_LABEL_NOP "nop"#endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" JUMP_LABEL_NOP "\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".word 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes);return false; l_yes:
arch/arm64/include/asm/jump_label.h +16 -2
@@ -26,14 +26,28 @@#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm goto("1: nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".align 3\n\t" ".quad 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : :"i" (key) : : l_yes); + : :"i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm goto("1: b %l[l_yes]\n\t" +".pushsection __jump_table, \"aw\"\n\t" +".align 3\n\t" +".quad 1b, %l[l_yes], %c0\n\t" +".popsection\n\t" + : :"i" (&((char *)key)[branch]) : : l_yes);return false; l_yes:
arch/mips/include/asm/jump_label.h +17 -2
@@ -26,14 +26,29 @@#define NOP_INSN "nop"#endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\t" NOP_INSN "\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" WORD_INSN " 1b, %l[l_yes], %0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\tj %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + WORD_INSN " 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
arch/powerpc/include/asm/jump_label.h +17 -2
@@ -18,14 +18,29 @@#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)#define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
arch/s390/include/asm/jump_label.h +17 -2
@@ -12,14 +12,29 @@* We use a brcl 0,2 instruction for jump labels at compile time so it * can be easily distinguished from a hotpatch generated instruction. */ -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n" ".pushsection __jump_table, \"aw\"\n" ".balign 8\n" ".quad 0b, %l[label], %0\n" ".popsection\n" - : : "X" (key) : : label); + : : "X" (&((char *)key)[branch]) : : label); + +return false; +label: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: j %l[l_yes]\n" + ".pushsection __jump_table, \"aw\"\n" + ".balign 8\n" + ".quad 0b, %l[label], %0\n" + ".popsection\n" + : : "X" (&((char *)key)[branch]) : : label); +return false; label:return true;
arch/sparc/include/asm/jump_label.h +25 -9
@@ -7,16 +7,32 @@#define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm_volatile_goto("1:\n\t" - "nop\n\t" - "nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 4\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); + asm_volatile_goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
arch/x86/include/asm/jump_label.h +19 -2
@@ -16,7 +16,7 @@# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC#endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
@@ -24,7 +24,24 @@ static __always_inline bool arch_static_ _ASM_ALIGN "\n\t" _ASM_PTR "1b, %l[l_yes], %c0 \n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:" + ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" + "2:\n\t" + ".pushsection __jump_table, \"aw\" \n\t" + _ASM_ALIGN "\n\t" + _ASM_PTR "1b, %l[l_yes], %c0 \n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
include/linux/jump_label.h +137 -6
167 lines not on this page · raw message
kernel/jump_label.c +23 -4
@@ -165,16 +165,32 @@ static inline bool static_key_type(strucstatic inline struct static_key *jump_entry_key (struct jump_entry *entry) { -return (struct static_key *)((unsigned long )entry->key); +return (struct static_key *)((unsigned long )entry->key & ~1UL ); +} + +static bool jump_entry_branch (struct jump_entry *entry) +{ +return (unsigned long )entry->key &1UL ; }static enum jump_label_type jump_label_type (struct jump_entry *entry) {struct static_key *key =jump_entry_key (entry);bool enabled =static_key_enabled (key); +bool branch =jump_entry_branch (entry); + +/* See the comment in linux/jump_label.h */ +return enabled ^ branch; +} + +static enum jump_label_type jump_label_init_type (struct jump_entry *entry) +{ +struct static_key *key =jump_entry_key (entry);bool type =static_key_type (key); +bool branch =jump_entry_branch (entry); -return enabled ^ type; +/* See the comment in linux/jump_label.h */ +return type ^ branch; } static void __jump_label_update(struct static_key *key,
@@ -276,8 +292,11 @@ void jump_label_apply_nops(struct moduleif (iter_start == iter_stop)return ; -for (iter = iter_start; iter < iter_stop; iter++) -arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); +for (iter = iter_start; iter < iter_stop; iter++) { +/* Only write NOPs for arch_branch_static(). */ +if (jump_label_init_type (iter) ==JUMP_LABEL_NOP ) +arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); + } }static int jump_label_add_module (struct module *mod)
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
arch/sparc/include/asm/jump_label.h line 14
Quoted 12 lines · Peter Zijlstra
arch/sparc/include/asm/jump_label.h +1
-4@@ -7,16 +7,32 @@#define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { -asm_volatile_goto ("1:\n\t" -"nop\n\t" - "nop\n\t"
It just occurred to me that this second nop is a branch delay slot, and therefore..
Quoted 13 lines
arch/sparc/include/asm/jump_label.h +8
-5- ".pushsection __jump_table, \"aw\"\n\t" - ".align 4\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); + asm_volatile_goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes);
arch/sparc/include/asm/jump_label.h line 29
Quoted 10 lines
arch/sparc/include/asm/jump_label.h +10
-0+ +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ +asm_volatile_goto ("1:\n\t" + "b %l[l_yes]\n\t"
should very much be here too:
"nop\n\t"
Quoted 9 lines
arch/sparc/include/asm/jump_label.h +6
-0+ ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
In reply to Peter ZijlstraQuoted 15 lines · Peter Zijlstra
arch/s390/include/asm/jump_label.h +1
-2@@ -12,14 +12,29 @@* We use a brcl 0,2 instruction for jump labels at compile time so it * can be easily distinguished from a hotpatch generated instruction. */ -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"".pushsection __jump_table, \"aw\"\n" ".balign 8\n" ".quad 0b, %l[label], %0\n" ".popsection\n" - : : "X" (key) : : label);
arch/s390/include/asm/jump_label.h line 31
Quoted 10 lines
arch/s390/include/asm/jump_label.h +10
-0+ : : "X" (&((char *)key)[branch]) : : label); + +return false; +label: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: j %l[l_yes]\n"
Looking at the s390 version of jump_label_make_branch(), this should have been:
"brcl 15, %l[l_yes]\n"
I suppose?
Quoted 9 lines
arch/s390/include/asm/jump_label.h +6
-0+ ".pushsection __jump_table, \"aw\"\n" + ".balign 8\n" + ".quad 0b, %l[label], %0\n" + ".popsection\n" + : : "X" (&((char *)key)[branch]) : : label); +return false; label:return true;
Re: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
Quoted 10 lines · Peter Zijlstra
arch/s390/include/asm/jump_label.h line 31
Quoted 3 lines
arch/s390/include/asm/jump_label.h +3
-0+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: j %l[l_yes]\n"Looking at the s390 version of
jump_label_make_branch(), this should have been:"brcl 15, %l[l_yes]\n"I suppose?
Yes. I wanted to test your version, but I assume you will send an updated version soon?
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
Quoted 8 lines · Heiko Carstens
Quoted 7 lines · Peter Zijlstra
arch/s390/include/asm/jump_label.h line 31
Quoted 3 lines
arch/s390/include/asm/jump_label.h +3
-0+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: j %l[l_yes]\n"Looking at the s390 version of
jump_label_make_branch(), this should have been:
Quoted 6 lines
Quoted 3 lines
"brcl 15, %l[l_yes]\n"I suppose?
Yes. I wanted to test your version, but I assume you will send an updated version soon?
I just send out updated patches for 5/6. They boot without issue on my x86_64.
Much obliged if you have a peek at that.
Thanks!
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Heiko Carstens
Quoted 9 lines · Peter Zijlstra
arch/s390/include/asm/jump_label.h line 31
Quoted 3 lines
arch/s390/include/asm/jump_label.h +3
-0+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: j %l[l_yes]\n"Looking at the s390 version of
jump_label_make_branch(), this should have been:"brcl 15, %l[l_yes]\n"
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
I suppose?
Yes. I wanted to test your version, but I assume you will send an updated version soon?
I just send out updated patches for 5/6. They boot without issue on my
x86_64.
Also, it helps if you convert one static_key user into something that'll trigger this new code.
The one I picked is unfortunately rather x86 specific.
Re: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
Quoted 17 lines · Peter Zijlstra
Quoted 16 lines · Peter Zijlstra
Quoted 14 lines · Heiko Carstens
Quoted 10 lines · Peter Zijlstra
arch/s390/include/asm/jump_label.h line 31
Quoted 3 lines
arch/s390/include/asm/jump_label.h +3
-0+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: j %l[l_yes]\n"Looking at the s390 version of
jump_label_make_branch(), this should have been:"brcl 15, %l[l_yes]\n"I suppose?
Yes. I wanted to test your version, but I assume you will send an updated version soon?
Quoted 7 lines
Quoted 2 lines
I just send out updated patches for 5/6. They boot without issue on my
x86_64.Also, it helps if you convert one
static_keyuser into something that'll trigger this new code.The one I picked is unfortunately rather x86 specific.
Yeah, just did that and it crashes ;)
Hopefully I'll have time to look into it today.
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
Quoted 2 lines · Heiko Carstens
Yeah, just did that and it crashes ;)
Hopefully I'll have time to look into it today.
*sigh*, sorry :/
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
In reply to Heiko CarstensQuoted 1 line · Heiko Carstens
Yeah, just did that and it crashes ;)
Does this make it go?
---
kernel/jump_label.c +4 -1
@@ -221,7 +221,10 @@ void __init jump_label_init(void)for (iter = iter_start; iter < iter_stop; iter++) {struct static_key *iterk; -arch_jump_label_transform_static (iter,jump_label_type (iter)); +/* rewrite NOPs */ +if (jump_label_type (iter) ==JUMP_LABEL_NOP ) +arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); + iterk =jump_entry_key (iter);if (iterk == key)continue ;
Peter Zijlstra
maintainerRe: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
In reply to Peter ZijlstraChanges:
- Fixes jump_label_add_module()
- Restored SPARC branch delay slot
- Fixed S/390 jump: brcl 15, %l
- Fixed ARM THUMB2: WASM(nop/b)
Quoted 1 line
From a quick reading of the microMIPS instruction reference it looks
like 'j' is still a 32bit instruction, albeit with a different encoding. The 16bit instruction is called 'j16'.
---
Subject: jump_label: Add a new static_key interface
From: Peter Zijlstra <[email protected]>
Date: Fri Jul 24 15:09:55 CEST 2015
There are various problems and short-comings with the current static_key interface:
- static_key_{true,false}() read like a branch depending on the key value, instead of the actual likely/unlikely branch depending on init value.
- static_key_{true,false}() are, as stated above, tied to the static_key init values STATIC_KEY_INIT_{TRUE,FALSE}.
- we're limited to the 2 (out of 4) possible options that compile to a default NOP because that's what our arch_static_branch() assembly emits.
So provide a new static_key interface:
DEFINE_STATIC_KEY_TRUE(name); DEFINE_STATIC_KEY_FALSE(name);
Which define a key of different types with an initial true/false value.
Then allow:
static_branch_likely()
static_branch_unlikely()
to take a key of either type and emit the right instruction for the case.
This means adding a second arch_static_branch_jump() assembly helper which emits a JMP per default.
In order to determine the right instruction for the right state, encode the branch type in the LSB of jump_entry::key.
--- arch/arm/include/asm/jump_label.h | 24 +++-- arch/arm64/include/asm/jump_label.h | 18 +++- arch/mips/include/asm/jump_label.h | 19 ++++ arch/powerpc/include/asm/jump_label.h | 19 ++++ arch/s390/include/asm/jump_label.h | 19 ++++ arch/sparc/include/asm/jump_label.h | 35 ++++++-- arch/x86/include/asm/jump_label.h | 21 ++++ include/linux/jump_label.h | 143 ++++++++++++++++++++++++++++++++-- kernel/jump_label.c | 30 +++++-- 9 files changed, 290 insertions(+), 38 deletions(-)
arch/arm/include/asm/jump_label.h +16 -8
@@ -7,20 +7,28 @@#define JUMP_LABEL_NOP_SIZE 4 -#ifdef CONFIG_THUMB2_KERNEL -#define JUMP_LABEL_NOP "nop.w" -#else -#define JUMP_LABEL_NOP "nop" -#endif +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + WASM(nop) "\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" - JUMP_LABEL_NOP "\n\t" + WASM(b) " %l[l_yes]\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".word 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes);return false; l_yes:
arch/arm64/include/asm/jump_label.h +16 -2
@@ -26,14 +26,28 @@#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm goto("1: nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".align 3\n\t" ".quad 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : :"i" (key) : : l_yes); + : :"i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm goto("1: b %l[l_yes]\n\t" +".pushsection __jump_table, \"aw\"\n\t" +".align 3\n\t" +".quad 1b, %l[l_yes], %c0\n\t" +".popsection\n\t" + : :"i" (&((char *)key)[branch]) : : l_yes);return false; l_yes:
arch/mips/include/asm/jump_label.h +17 -2
@@ -26,14 +26,29 @@#define NOP_INSN "nop"#endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\t" NOP_INSN "\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" WORD_INSN " 1b, %l[l_yes], %0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\tj %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + WORD_INSN " 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
arch/powerpc/include/asm/jump_label.h +17 -2
@@ -18,14 +18,29 @@#define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG)#define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
arch/s390/include/asm/jump_label.h +17 -2
@@ -12,14 +12,29 @@* We use a brcl 0,2 instruction for jump labels at compile time so it * can be easily distinguished from a hotpatch generated instruction. */ -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n" ".pushsection __jump_table, \"aw\"\n" ".balign 8\n" ".quad 0b, %l[label], %0\n" ".popsection\n" - : : "X" (key) : : label); + : : "X" (&((char *)key)[branch]) : : label); + +return false; +label: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: brcl 15, %l[l_yes]\n" + ".pushsection __jump_table, \"aw\"\n" + ".balign 8\n" + ".quad 0b, %l[label], %0\n" + ".popsection\n" + : : "X" (&((char *)key)[branch]) : : label); +return false; label:return true;
arch/sparc/include/asm/jump_label.h +26 -9
@@ -7,16 +7,33 @@#define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm_volatile_goto("1:\n\t" - "nop\n\t" - "nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 4\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); + asm_volatile_goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
arch/x86/include/asm/jump_label.h +19 -2
@@ -16,7 +16,7 @@# define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC#endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
@@ -24,7 +24,24 @@ static __always_inline bool arch_static_ _ASM_ALIGN "\n\t" _ASM_PTR "1b, %l[l_yes], %c0 \n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + +return false; +l_yes: +return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:" + ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" + "2:\n\t" + ".pushsection __jump_table, \"aw\" \n\t" + _ASM_ALIGN "\n\t" + _ASM_PTR "1b, %l[l_yes], %c0 \n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); +return false; l_yes:return true;
include/linux/jump_label.h +137 -6
167 lines not on this page · raw message
kernel/jump_label.c +25 -5
@@ -165,16 +165,32 @@ static inline bool static_key_type(strucstatic inline struct static_key *jump_entry_key (struct jump_entry *entry) { -return (struct static_key *)((unsigned long )entry->key); +return (struct static_key *)((unsigned long )entry->key & ~1UL ); +} + +static bool jump_entry_branch (struct jump_entry *entry) +{ +return (unsigned long )entry->key &1UL ; }static enum jump_label_type jump_label_type (struct jump_entry *entry) {struct static_key *key =jump_entry_key (entry);bool enabled =static_key_enabled (key); +bool branch =jump_entry_branch (entry); + +/* See the comment in linux/jump_label.h */ +return enabled ^ branch; +} + +static enum jump_label_type jump_label_init_type (struct jump_entry *entry) +{ +struct static_key *key =jump_entry_key (entry);bool type =static_key_type (key); +bool branch =jump_entry_branch (entry); -return enabled ^ type; +/* See the comment in linux/jump_label.h */ +return type ^ branch; } static void __jump_label_update(struct static_key *key,
@@ -276,8 +292,11 @@ void jump_label_apply_nops(struct moduleif (iter_start == iter_stop)return ; -for (iter = iter_start; iter < iter_stop; iter++) -arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); +for (iter = iter_start; iter < iter_stop; iter++) { +/* Only write NOPs for arch_branch_static(). */ +if (jump_label_init_type (iter) ==JUMP_LABEL_NOP ) +arch_jump_label_transform_static (iter,JUMP_LABEL_NOP ); + } }static int jump_label_add_module (struct module *mod)
@@ -318,7 +337,8 @@ static int jump_label_add_module(struct jlm->next = key->next; key->next = jlm; -if (jump_label_type (iter) ==JUMP_LABEL_JMP ) +/* Only update if we've changed from our initial state */ +if (jump_label_type (iter) !=jump_label_init_type (iter))__jump_label_update (key, iter, iter_stop); }
Re: [RFC][PATCH 6/7] jump_label: Add a new static_key interface
arch/s390/include/asm/jump_label.h line 36
Quoted 10 lines · Peter Zijlstra
arch/s390/include/asm/jump_label.h +8
-0+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: brcl 15, %l[l_yes]\n" + ".pushsection __jump_table, \"aw\"\n" + ".balign 8\n" + ".quad 0b, %l[label], %0\n" + ".popsection\n" + : : "X" (&((char *)key)[branch]) : : label);
The above should have been
asm_volatile_goto("0: brcl 15,%l[label]\n"(label instead of l_yes). Just recognized this, sorry..
Peter Zijlstra
maintainerRFC PATCH 7/7 x86, tsc: Employ static_branch_likely()
In reply to Peter ZijlstraBecause of the static_key restrictions we had to take an unconditional jump for the most likely case, causing $I bloat.
Rewrite to use the new primitives.
--- arch/x86/kernel/tsc.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-)
arch/x86/kernel/tsc.c +10 -12
@@ -38,7 +38,7 @@ static int __read_mostly tsc_unstable;erroneous rdtsc usage on !cpu_has_tsc processors */ static int __read_mostly tsc_disabled =-1 ; -static struct static_key __use_tsc =STATIC_KEY_INIT ; +static DEFINE_STATIC_KEY_FALSE (__use_tsc);int tsc_clocksource_reliable;
@@ -274,7 +274,12 @@ static void set_cyc2ns_scale(unsigned lo*/ u64 native_sched_clock (void ) { -u64 tsc_now; +if (static_branch_likely (&__use_tsc)) { +u64 tsc_now =rdtsc (); + +/* return the value in ns */ +return cycles_2_ns (tsc_now); + }/* * Fall back to jiffies if there's no TSC available:
@@ -284,16 +289,9 @@ u64 native_sched_clock(void)* very important for it to be as fast as the platform * can achieve it. ) */ -if (!static_key_false (&__use_tsc)) { -/* No locking but a rare wrong value is not a big deal: */ -return (jiffies_64 -INITIAL_JIFFIES ) * (1000000000 /HZ ); - } - -/* read the Time Stamp Counter: */ - tsc_now =rdtsc (); -/* return the value in ns */ -return cycles_2_ns (tsc_now); +/* No locking but a rare wrong value is not a big deal: */ +return (jiffies_64 -INITIAL_JIFFIES ) * (1000000000 /HZ ); }/* We need to define a real function for sched_clock, to override the
@@ -1204,7 +1202,7 @@ void __init tsc_init(void)/* now allow native_sched_clock() to use rdtsc */ tsc_disabled =0 ; -static_key_slow_inc (&__use_tsc); +static_branch_enable (&__use_tsc);if (!no_sched_irq_time)enable_sched_clock_irqtime ();