perf fixes
Subsystems (4)
Linus,
Please pull the latest perf-fixes-for-linus git tree from:
git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip.git perf-fixes-for-linus
One of the fixes works around a compiler bug in the jump label optimization on x86-32 - and while this fixes the bug it reduced our confidence in GCC getting jump labels right, so we also made the optimization Kconfig optional (and it defaults to off).
Thanks,
Ingo
------------------>
David Miller (1):
jump_label: Fix unaligned traps on sparc.
H. Peter Anvin (1):
x86, ftrace: Use safe noops, drop trap test
Jason Baron (2):
jump label: Fix module __init section race
jump label: Fix deadlock b/w jump_label_mutex vs. text_mutex
Santosh Shilimkar (1):
oprofile: Fix the hang while taking the cpu offline
Steven Rostedt (3):
jump label: Fix error with preempt disable holding mutex jump label: Make arch_jump_label_text_poke_early() optional jump label: Add work around to i386 gcc asm goto bug
Tejun Heo (1):
oprofile: Remove deprecated use of flush_scheduled_work()
arch/Kconfig | 14 ++++++ arch/sparc/include/asm/jump_label.h | 1 + arch/x86/Makefile_32.cpu | 13 +++++- arch/x86/kernel/alternative.c | 69 +++++++------------------------ drivers/oprofile/buffer_sync.c | 2 +- drivers/oprofile/cpu_buffer.c | 10 +++- drivers/oprofile/cpu_buffer.h | 1 + drivers/oprofile/timer_int.c | 13 ++++++ include/linux/jump_label.h | 7 +++- kernel/jump_label.c | 77 ++++++++++++++++++++++++++++++----- kernel/kprobes.c | 26 +++++++----- 11 files changed, 152 insertions(+), 81 deletions(-)
arch/Kconfig +14 -0
@@ -42,6 +42,20 @@ config KPROBES for kernel debugging, non-intrusive instrumentation and testing. If in doubt, say "N". +config JUMP_LABEL + bool "Optimize trace point call sites" + depends on HAVE_ARCH_JUMP_LABEL + help + If it is detected that the compiler has support for "asm goto", + the kernel will compile trace point locations with just a + nop instruction. When trace points are enabled, the nop will + be converted to a jump to the trace function. This technique + lowers overhead and stress on the branch prediction of the + processor. + + On i386, options added to the compiler flags may increase + the size of the kernel slightly. + config OPTPROBES def_bool y depends on KPROBES && HAVE_OPTPROBES
arch/sparc/include/asm/jump_label.h +1 -0
@@ -14,6 +14,7 @@"nop\n\t" \"nop\n\t" \".pushsection __jump_table, \"a\"\n\t" \ +".align 4\n\t" \ ".word 1b, %l[" #label "], %c0\n\t" \ ".popsection \n\t" \ : : "i" (key) : : label);\
arch/x86/Makefile_32.cpu +12 -1
@@ -51,7 +51,18 @@ cflags-$(CONFIG_X86_GENERIC) += $(call tune,generic,$(call tune,i686)) # prologue (push %ebp, mov %esp, %ebp) which breaks the function graph # tracer assumptions. For i686, generic, core2 this is set by the # compiler anyway -cflags-$(CONFIG_FUNCTION_GRAPH_TRACER) += $(call cc-option,-maccumulate-outgoing-args) +ifeq ($(CONFIG_FUNCTION_GRAPH_TRACER), y) +ADD_ACCUMULATE_OUTGOING_ARGS := y +endif + +# Work around to a bug with asm goto with first implementations of it +# in gcc causing gcc to mess up the push and pop of the stack in some +# uses of asm goto. +ifeq ($(CONFIG_JUMP_LABEL), y) +ADD_ACCUMULATE_OUTGOING_ARGS := y +endif + +cflags-$(ADD_ACCUMULATE_OUTGOING_ARGS) += $(call cc-option,-maccumulate-outgoing-args) # Bug fix for binutils: this option is required in order to keep # binutils from generating NOPL instructions against our will.
arch/x86/kernel/alternative.c +15 -54
80 lines not on this page · raw message
drivers/oprofile/buffer_sync.c +1 -1
@@ -190,7 +190,7 @@ void sync_stop(void)profile_event_unregister (PROFILE_TASK_EXIT , &task_exit_nb);task_handoff_unregister (&task_free_nb);mutex_unlock (&buffer_mutex); -flush_scheduled_work (); +flush_cpu_work ();/* make sure we don't leak task structs */ process_task_mortuary ();
drivers/oprofile/cpu_buffer.c +7 -3
@@ -111,14 +111,18 @@ void start_cpu_work(void)void end_cpu_work (void ) { -int i; - work_enabled =0 ; +} + +void flush_cpu_work (void ) +{ +int i;for_each_online_cpu (i) {struct oprofile_cpu_buffer *b = &per_cpu (op_cpu_buffer, i); -cancel_delayed_work (&b->work); +/* these works are per-cpu, no need for flush_sync */ +flush_delayed_work (&b->work); } }
drivers/oprofile/cpu_buffer.h +1 -0
@@ -25,6 +25,7 @@ void free_cpu_buffers(void);void start_cpu_work (void );void end_cpu_work (void ); +void flush_cpu_work (void );/* CPU buffer is composed of such entries (which are * also used for context switch notes)
drivers/oprofile/timer_int.c +13 -0
@@ -21,6 +21,7 @@#include "oprof.h" static DEFINE_PER_CPU(struct hrtimer, oprofile_hrtimer); +static int ctr_running;static enum hrtimer_restart oprofile_hrtimer_notify (struct hrtimer *hrtimer) {
@@ -33,6 +34,9 @@ static void __oprofile_hrtimer_start(void *unused) {struct hrtimer *hrtimer = &__get_cpu_var (oprofile_hrtimer); +if (!ctr_running) +return ; +hrtimer_init (hrtimer,CLOCK_MONOTONIC ,HRTIMER_MODE_REL ); hrtimer->function = oprofile_hrtimer_notify;
@@ -42,7 +46,10 @@ static void __oprofile_hrtimer_start(void *unused)static int oprofile_hrtimer_start (void ) { +get_online_cpus (); + ctr_running =1 ;on_each_cpu (__oprofile_hrtimer_start,NULL ,1 ); +put_online_cpus ();return 0 ; }
@@ -50,6 +57,9 @@ static void __oprofile_hrtimer_stop(int cpu) {struct hrtimer *hrtimer = &per_cpu (oprofile_hrtimer, cpu); +if (!ctr_running) +return ; +hrtimer_cancel (hrtimer); }
@@ -57,8 +67,11 @@ static void oprofile_hrtimer_stop(void) {int cpu; +get_online_cpus ();for_each_online_cpu (cpu)__oprofile_hrtimer_stop (cpu); + ctr_running =0 ; +put_online_cpus (); } static int __cpuinit oprofile_cpu_notify(struct notifier_block *self,
include/linux/jump_label.h +6 -1
@@ -1,7 +1,7 @@#ifndef _LINUX_JUMP_LABEL_H#define _LINUX_JUMP_LABEL_H -#if defined(CC_HAVE_ASM_GOTO ) && defined(CONFIG_HAVE_ARCH_JUMP_LABEL ) +#if defined(CC_HAVE_ASM_GOTO ) && defined(CONFIG_JUMP_LABEL )# include <asm/jump_label.h> # define HAVE_JUMP_LABEL #endif
@@ -18,6 +18,8 @@ struct module;extern struct jump_entry __start___jump_table[];extern struct jump_entry __stop___jump_table[]; +extern void jump_label_lock (void ); +extern void jump_label_unlock (void );extern void arch_jump_label_transform (struct jump_entry *entry,enum jump_label_type type);extern void arch_jump_label_text_poke_early (jump_label_t addr);
@@ -59,6 +61,9 @@ static inline int jump_label_text_reserved(void *start, void *end) return 0; } +static inline void jump_label_lock (void ) {} +static inline void jump_label_unlock (void ) {} +#endif #define COND_STMT (key, stmt) \
kernel/jump_label.c +66 -11
151 lines not on this page · raw message
kernel/kprobes.c +16 -10
@@ -1145,14 +1145,13 @@ int __kprobes register_kprobe(struct kprobe *p)if (ret)return ret; +jump_label_lock ();preempt_disable ();if (!kernel_text_address ((unsigned long ) p->addr) ||in_kprobes_functions ((unsigned long ) p->addr) ||ftrace_text_reserved (p->addr, p->addr) || -jump_label_text_reserved (p->addr, p->addr)) { -preempt_enable (); -return -EINVAL ; - } +jump_label_text_reserved (p->addr, p->addr)) + goto fail_with_jump_label;/* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */ p->flags &=KPROBE_FLAG_DISABLED ;
@@ -1166,10 +1165,9 @@ int __kprobes register_kprobe(struct kprobe *p)* We must hold a refcount of the probed module while updating * its code to prohibit unexpected unloading. */ -if (unlikely (!try_module_get (probed_mod))) { -preempt_enable (); -return -EINVAL ; - } +if (unlikely (!try_module_get (probed_mod))) + goto fail_with_jump_label; +/* * If the module freed .init.text, we couldn't insert * kprobes in there.
@@ -1177,16 +1175,18 @@ int __kprobes register_kprobe(struct kprobe *p)if (within_module_init ((unsigned long )p->addr, probed_mod) && probed_mod->state !=MODULE_STATE_COMING ) {module_put (probed_mod); -preempt_enable (); -return -EINVAL ; + goto fail_with_jump_label; } }preempt_enable (); +jump_label_unlock (); p->nmissed =0 ;INIT_LIST_HEAD (&p->list);mutex_lock (&kprobe_mutex); +jump_label_lock ();/* needed to call jump_label_text_reserved() */ +get_online_cpus ();/* For avoiding text_mutex deadlock. */ mutex_lock (&text_mutex);
@@ -1214,12 +1214,18 @@ int __kprobes register_kprobe(struct kprobe *p) out:mutex_unlock (&text_mutex);put_online_cpus (); +jump_label_unlock ();mutex_unlock (&kprobe_mutex);if (probed_mod)module_put (probed_mod);return ret; + +fail_with_jump_label: +preempt_enable (); +jump_label_unlock (); +return -EINVAL ; }EXPORT_SYMBOL_GPL (register_kprobe);