make Jump Labels work early
Subsystems (4)
Hi all,
This is the series which makes jump labels work early. Does someone want to pick this up, or should I just send it to Linus directly?
[ I see Linus has already picked up the stop-machine patch from -mm. ]
Thanks,
J
The following changes since commit a102a9ece5489e1718cd7543aa079082450ac3a2:
Linux 3.1-rc8 (2011-09-27 15:48:34 -0700)
are available in the git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/jeremy/xen.git upstream/jump-label-noearly
Jeremy Fitzhardinge (9):
jump_label: use proper atomic_t initializer
stop_machine: make stop_machine safe and efficient to call early
jump_label: if a key has already been initialized, don't nop it out x86/jump_label: drop arch_jump_label_text_poke_early() sparc/jump_label: drop arch_jump_label_text_poke_early()
jump_label: add arch_jump_label_transform_static() to optimise non-live code updates s390/jump-label: add arch_jump_label_transform_static() x86/jump_label: add arch_jump_label_transform_static()
jump-label: initialize jump-label subsystem much earlier
arch/s390/kernel/jump_label.c | 51 ++++++++++++++++++++++++--------------- arch/sparc/kernel/jump_label.c | 8 ------ arch/x86/kernel/jump_label.c | 20 +++++++++++---- include/linux/jump_label.h | 23 +++++++++++------- init/main.c | 3 ++ kernel/jump_label.c | 37 ++++++++++++++++------------ kernel/stop_machine.c | 22 +++++++++++++++++ 7 files changed, 105 insertions(+), 59 deletions(-)
arch/s390/kernel/jump_label.c +31 -20
@@ -18,26 +18,15 @@ struct insn { } __packed;struct insn_args { -unsigned long *target; -struct insn *insn; -ssize_t size; +struct jump_entry *entry; +enum jump_label_type type; }; -static int __arch_jump_label_transform (void *data) +static void __jump_label_transform (struct jump_entry *entry, +enum jump_label_type type) { -struct insn_args *args = data; -int rc; - - rc =probe_kernel_write (args->target, args->insn, args->size); -WARN_ON_ONCE (rc <0 ); -return 0 ; -} - -void arch_jump_label_transform (struct jump_entry *entry, -enum jump_label_type type) -{ -struct insn_args args;struct insn insn; +int rc;if (type ==JUMP_LABEL_ENABLE ) {/* brcl 15,offset */
@@ -49,11 +38,33 @@ void arch_jump_label_transform(struct jump_entry *entry, insn.offset = 0; } - args.target = (void *) entry->code; - args.insn = &insn; - args.size =JUMP_LABEL_NOP_SIZE ; + rc =probe_kernel_write ((void *)entry->code, &insn,JUMP_LABEL_NOP_SIZE ); +WARN_ON_ONCE (rc <0 ); +} -stop_machine (__arch_jump_label_transform, &args,NULL ); +static int __sm_arch_jump_label_transform (void *data) +{ +struct insn_args *args = data; + +__jump_label_transform (args->entry, args->type); +return 0 ; +} + +void arch_jump_label_transform (struct jump_entry *entry, +enum jump_label_type type) +{ +struct insn_args args; + + args.entry = entry; + args.type = type; + +stop_machine (__sm_arch_jump_label_transform, &args,NULL ); +} + +void arch_jump_label_transform_static (struct jump_entry *entry, +enum jump_label_type type) +{ +__jump_label_transform (entry, type); }#endif
arch/sparc/kernel/jump_label.c +0 -8
@@ -36,12 +36,4 @@ void arch_jump_label_transform(struct jump_entry *entry, put_online_cpus(); } -void arch_jump_label_text_poke_early (jump_label_t addr) -{ -u32 *insn_p = (u32 *) (unsigned long ) addr; - - *insn_p =0x01000000 ; -flushi (insn_p); -} -#endif
arch/x86/kernel/jump_label.c +14 -6
@@ -24,8 +24,9 @@ union jump_code_union { } __attribute__((packed)); }; -void arch_jump_label_transform (struct jump_entry *entry, -enum jump_label_type type) +static void __jump_label_transform (struct jump_entry *entry, +enum jump_label_type type, +void *(*poker)(void *,const void *,size_t )) {union jump_code_union code;
@@ -35,17 +36,24 @@ void arch_jump_label_transform(struct jump_entry *entry, (entry->code + JUMP_LABEL_NOP_SIZE); }else memcpy (&code, ideal_nops[NOP_ATOMIC5 ],JUMP_LABEL_NOP_SIZE ); + + (*poker)((void *)entry->code, &code,JUMP_LABEL_NOP_SIZE ); +} + +void arch_jump_label_transform (struct jump_entry *entry, +enum jump_label_type type) +{get_online_cpus ();mutex_lock (&text_mutex); -text_poke_smp ((void *)entry->code, &code,JUMP_LABEL_NOP_SIZE ); +__jump_label_transform (entry, type, text_poke_smp);mutex_unlock (&text_mutex);put_online_cpus (); } -void arch_jump_label_text_poke_early (jump_label_t addr) +void arch_jump_label_transform_static (struct jump_entry *entry, +enum jump_label_type type) { -text_poke_early ((void *)addr, ideal_nops[NOP_ATOMIC5 ], -JUMP_LABEL_NOP_SIZE ); +__jump_label_transform (entry, type, text_poke_early); }#endif
include/linux/jump_label.h +14 -9
@@ -16,7 +16,7 @@ struct jump_label_key {# include <asm/jump_label.h> # define HAVE_JUMP_LABEL -#endif +#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ enum jump_label_type {JUMP_LABEL_DISABLE =0 ,
@@ -28,9 +28,9 @@ struct module;#ifdef HAVE_JUMP_LABEL #ifdef CONFIG_MODULES -#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL} +#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}#else -#define JUMP_LABEL_INIT {{ 0 }, NULL} +#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}#endif static __always_inline bool static_branch(struct jump_label_key *key)
@@ -41,18 +41,20 @@ static __always_inline bool static_branch(struct jump_label_key *key)extern struct jump_entry __start___jump_table[];extern struct jump_entry __stop___jump_table[]; +extern void jump_label_init (void );extern void jump_label_lock (void );extern void jump_label_unlock (void );extern void arch_jump_label_transform (struct jump_entry *entry, -enum jump_label_type type); -extern void arch_jump_label_text_poke_early (jump_label_t addr); +enum jump_label_type type); +extern void arch_jump_label_transform_static (struct jump_entry *entry, +enum jump_label_type type);extern int jump_label_text_reserved (void *start,void *end);extern void jump_label_inc (struct jump_label_key *key);extern void jump_label_dec (struct jump_label_key *key);extern bool jump_label_enabled (struct jump_label_key *key);extern void jump_label_apply_nops (struct module *mod); -#else +#else /* !HAVE_JUMP_LABEL */ #include <linux/atomic.h>
@@ -62,6 +64,10 @@ struct jump_label_key {atomic_t enabled; }; +static __always_inline void jump_label_init(void) +{ +} + static __always_inline bool static_branch(struct jump_label_key *key) {if (unlikely (atomic_read (&key->enabled)))
@@ -96,7 +102,6 @@ static inline int jump_label_apply_nops(struct module *mod) {return 0 ; } +#endif /* HAVE_JUMP_LABEL */ -#endif - -#endif +#endif /* _LINUX_JUMP_LABEL_H */
init/main.c +3 -0
@@ -515,6 +515,9 @@ asmlinkage void __init start_kernel(void)parse_args ("Booting kernel" , static_command_line, __start___param, __stop___param - __start___param, &unknown_bootoption); + +jump_label_init (); +/* * These use large bootmem allocations and must precede * kmem_cache_init()
kernel/jump_label.c +21 -16
@@ -104,6 +104,18 @@ static int __jump_label_text_reserved(struct jump_entry *iter_start,return 0 ; } +/* +* Update code which is definitely not currently executing. +* Architectures which need heavyweight synchronization to modify +* running code can override this to make the non-live update case +* cheaper. +*/ +void __weak arch_jump_label_transform_static (struct jump_entry *entry, +enum jump_label_type type) +{ +arch_jump_label_transform (entry, type); +} +static void __jump_label_update (struct jump_label_key *key,struct jump_entry *entry,struct jump_entry *stop,int enable)
@@ -121,14 +133,7 @@ static void __jump_label_update(struct jump_label_key *key, } } -/* -* Not all archs need this. -*/ -void __weak arch_jump_label_text_poke_early (jump_label_t addr) -{ -} - -static __init int jump_label_init (void ) +void __init jump_label_init (void ) {struct jump_entry *iter_start = __start___jump_table;struct jump_entry *iter_stop = __stop___jump_table;
@@ -139,22 +144,22 @@ static __init int jump_label_init(void)jump_label_sort_entries (iter_start, iter_stop);for (iter = iter_start; iter < iter_stop; iter++) { -arch_jump_label_text_poke_early (iter->code); -if (iter->key == (jump_label_t )(unsigned long )key) +struct jump_label_key *iterk; + + iterk = (struct jump_label_key *)(unsigned long )iter->key; +arch_jump_label_transform_static (iter,jump_label_enabled (iterk) ? +JUMP_LABEL_ENABLE :JUMP_LABEL_DISABLE ); +if (iterk == key)continue ; - key = (struct jump_label_key *)(unsigned long )iter->key; -atomic_set (&key->enabled,0 ); + key = iterk; key->entries = iter;#ifdef CONFIG_MODULES key->next =NULL ;#endif }jump_label_unlock (); - -return 0 ; } -early_initcall (jump_label_init);#ifdef CONFIG_MODULES
@@ -212,7 +217,7 @@ void jump_label_apply_nops(struct module *mod)return ;for (iter = iter_start; iter < iter_stop; iter++) -arch_jump_label_text_poke_early (iter->code); +arch_jump_label_transform_static (iter,JUMP_LABEL_DISABLE ); }static int jump_label_add_module (struct module *mod)
kernel/stop_machine.c +22 -0
@@ -41,6 +41,7 @@ struct cpu_stopper { }; static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper); +static bool stop_machine_initialized = false;static void cpu_stop_init_done (struct cpu_stop_done *done,unsigned int nr_todo) {
@@ -386,6 +387,8 @@ static int __init cpu_stop_init(void)cpu_stop_cpu_callback (&cpu_stop_cpu_notifier,CPU_ONLINE , bcpu);register_cpu_notifier (&cpu_stop_cpu_notifier); + stop_machine_initialized = true; +return 0 ; }early_initcall (cpu_stop_init);
@@ -485,6 +488,25 @@ int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus) .num_threads = num_online_cpus(), .active_cpus = cpus }; +if (!stop_machine_initialized) { +/* +* Handle the case where stop_machine() is called +* early in boot before stop_machine() has been +* initialized. +*/ +unsigned long flags; +int ret; + +WARN_ON_ONCE (smdata.num_threads !=1 ); + +local_irq_save (flags); +hard_irq_disable (); + ret = (*fn)(data); +local_irq_restore (flags); + +return ret; + } +/* Set the initial state and stop all online cpus. */ set_state (&smdata,STOPMACHINE_PREPARE );return stop_cpus (cpu_online_mask, stop_machine_cpu_stop, &smdata);