x86_64 fix idle notifier
PATCH 1/1 x86_64 fix idle notifier
Hello,
Here is a patch against Linus GIT tree that fixes the idle notifier for the X86-64 tree. With this notifiier you can get called when entering/exiting the core of the idle loop. I don't think it is used yet but I am planning to leverage it for perfmon.
There was an issue in that it would not capture all re-entries into the idle loop because of missing hooks on the return from interrupt path.
I have also modified the header file (idle.h) to avoid branching in case this is not needed. The enter_idle()/exit_idle() calls are not integrated into irq_enter()/irq_exit() because those are generic and for now the idle notifier is arch specific.
Changelog:
- fix the x86-64 idle notifier to cover all re-entries into the core idle loop after an interrupt.
- define enter_idle/exit_idle in idle.h to avoid branching when not needed
arch/i386/kernel/apic.c +6 -0
@@ -1230,9 +1230,11 @@ fastcall void smp_apic_timer_interrupt(s* Besides, if we don't timer interrupts ignore the global * interrupt lock, which is the WrongThing (tm) to do. */ +exit_idle ();irq_enter ();smp_local_timer_interrupt (regs);irq_exit (); +enter_idle (); }#ifndef CONFIG_SMP
@@ -1279,6 +1281,7 @@ fastcall void smp_spurious_interrupt(str {unsigned long v; +exit_idle ();irq_enter ();/* * Check if this really is a spurious interrupt and ACK it
@@ -1293,6 +1296,7 @@ fastcall void smp_spurious_interrupt(strprintk (KERN_INFO "spurious APIC interrupt on CPU#%d, should never happen.\n" ,smp_processor_id ());irq_exit (); +enter_idle (); }/*
@@ -1303,6 +1307,7 @@ fastcall void smp_error_interrupt(struct {unsigned long v, v1; +exit_idle ();irq_enter ();/* First tickle the hardware, only then report what went on. -- REW */ v =apic_read (APIC_ESR );
@@ -1324,6 +1329,7 @@ fastcall void smp_error_interrupt(structprintk (KERN_DEBUG "APIC error on CPU%d: %02lx(%02lx)\n" ,smp_processor_id (), v , v1);irq_exit (); +enter_idle (); }/*
arch/i386/kernel/cpu/mcheck/p4.c +2 -0
@@ -70,9 +70,11 @@ static void (*vendor_thermal_interrupt)( fastcall void smp_thermal_interrupt(struct pt_regs *regs) { +exit_idle ();irq_enter ();vendor_thermal_interrupt (regs);irq_exit (); +enter_idle (); }/* P4/Xeon Thermal regulation detect and init */
arch/i386/kernel/process.c +31 -0
@@ -78,6 +78,35 @@ void (*pm_idle)(void);EXPORT_SYMBOL (pm_idle); static DEFINE_PER_CPU(unsigned int, cpu_idle_state); +static ATOMIC_NOTIFIER_HEAD (idle_notifier); + +void idle_notifier_register (struct notifier_block *n) +{ +atomic_notifier_chain_register (&idle_notifier, n); +} +EXPORT_SYMBOL_GPL (idle_notifier_register); + +void idle_notifier_unregister (struct notifier_block *n) +{ +atomic_notifier_chain_unregister (&idle_notifier, n); +} +EXPORT_SYMBOL (idle_notifier_unregister); + +enum idle_state {CPU_IDLE ,CPU_NOT_IDLE }; +static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE; + +void __enter_idle (void ) +{ +__get_cpu_var (idle_state) =CPU_IDLE ; +atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); +} + +void __exit_idle (void ) +{ +__get_cpu_var (idle_state) =CPU_NOT_IDLE ; +atomic_notifier_call_chain (&idle_notifier,IDLE_END ,NULL ); +} +void disable_hlt (void ) { hlt_counter++;
@@ -193,7 +222,9 @@ void cpu_idle(void)play_dead ();__get_cpu_var (irq_stat).idle_timestamp = jiffies; +__enter_idle ();idle (); +__exit_idle (); }preempt_enable_no_resched ();schedule ();
arch/i386/kernel/smp.c +2 -0
@@ -624,6 +624,7 @@ fastcall void smp_call_function_interrup/* * At this point the info structure may be out of scope unless wait==1 */ +exit_idle ();irq_enter (); (*func)(info);irq_exit ();
@@ -632,5 +633,6 @@ fastcall void smp_call_function_interrupmb ();atomic_inc (&call_data->finished); } +enter_idle (); }
arch/x86_64/kernel/apic.c +3 -0
@@ -988,6 +988,7 @@ void smp_apic_timer_interrupt(struct pt_irq_enter ();smp_local_timer_interrupt (regs);irq_exit (); +enter_idle (); }/*
@@ -1069,6 +1070,7 @@ asmlinkage void smp_spurious_interrupt(v }#endif irq_exit(); +enter_idle (); }/*
@@ -1101,6 +1103,7 @@ asmlinkage void smp_error_interrupt(voidprintk (KERN_DEBUG "APIC error on CPU%d: %02x(%02x)\n" ,smp_processor_id (), v , v1);irq_exit (); +enter_idle (); }int disable_apic;
arch/x86_64/kernel/irq.c +1 -0
@@ -131,6 +131,7 @@ asmlinkage unsigned int do_IRQ(struct pt#endif __do_IRQ (irq, regs);irq_exit (); +enter_idle ();return 1 ; }
arch/x86_64/kernel/mce_amd.c +1 -0
@@ -224,6 +224,7 @@ asmlinkage void mce_threshold_interrupt( } out:irq_exit (); +enter_idle (); }/*
arch/x86_64/kernel/mce_intel.c +1 -0
@@ -42,6 +42,7 @@ asmlinkage void smp_thermal_interrupt(vomce_log (&m); done:irq_exit (); +enter_idle (); }static void __cpuinit intel_init_thermal (struct cpuinfo_x86 *c)
arch/x86_64/kernel/process.c +3 -11
@@ -83,26 +83,18 @@ EXPORT_SYMBOL(idle_notifier_unregister);enum idle_state {CPU_IDLE ,CPU_NOT_IDLE }; static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE; -void enter_idle (void ) +void __enter_idle (void ) {__get_cpu_var (idle_state) =CPU_IDLE ;atomic_notifier_call_chain (&idle_notifier,IDLE_START ,NULL ); } -static void __exit_idle (void ) +void __exit_idle (void ) {__get_cpu_var (idle_state) =CPU_NOT_IDLE ;atomic_notifier_call_chain (&idle_notifier,IDLE_END ,NULL ); } -/* Called from interrupts to signify idle end */ -void exit_idle (void ) -{ -if (current->pid |read_pda (irqcount)) -return ; -__exit_idle (); -} -/* * We use this if we don't have any better * idle routine..
@@ -218,7 +210,7 @@ void cpu_idle (void) idle = default_idle;if (cpu_is_offline (smp_processor_id ()))play_dead (); -enter_idle (); +__enter_idle ();idle ();__exit_idle (); }
arch/x86_64/kernel/smp.c +1 -0
@@ -520,6 +520,7 @@ asmlinkage void smp_call_function_interrmb ();atomic_inc (&call_data->finished); } +enter_idle (); }int safe_smp_processor_id (void )
include/asm-x86_64/idle.h +20 -2
@@ -8,7 +8,25 @@ struct notifier_block;void idle_notifier_register (struct notifier_block *n);void idle_notifier_unregister (struct notifier_block *n); -void enter_idle (void ); -void exit_idle (void ); +void __exit_idle (void ); +void __enter_idle (void ); + +/* Called from interrupts to signify back to idle */ +static inline void enter_idle (void ) +{ +/* bitwise or is intentional */ +if (current->pid |read_pda (irqcount)) +return ; +__enter_idle (); +} + +/* Called from interrupts to signify idle end */ +static inline void exit_idle (void ) +{ +/* bitwise or is intentional */ +if (current->pid |read_pda (irqcount)) +return ; +__exit_idle (); +}#endif