Unified NMI delayed call mechanism
Subsystems (2)
RFC 1/3 Unified NMI delayed call mechanism
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
--- arch/x86/include/asm/entry_arch.h | 1 arch/x86/include/asm/hw_irq.h | 1 arch/x86/include/asm/irq_vectors.h | 5 + arch/x86/include/asm/nmi.h | 7 ++ arch/x86/kernel/entry_64.S | 3 + arch/x86/kernel/irqinit.c | 3 + arch/x86/kernel/traps.c | 104 +++++++++++++++++++++++++++++++++++++ 7 files changed, 124 insertions(+)
arch/x86/include/asm/entry_arch.h +1 -0
@@ -65,4 +65,5 @@ BUILD_INTERRUPT(threshold_interrupt,THRE BUILD_INTERRUPT(mce_self_interrupt,MCE_SELF_VECTOR)#endif +BUILD_INTERRUPT(nmi_delayed_call_interrupt,NMI_DELAYED_CALL_VECTOR)#endif
arch/x86/include/asm/hw_irq.h +1 -0
@@ -35,6 +35,7 @@ extern void spurious_interrupt(void);extern void thermal_interrupt (void );extern void reschedule_interrupt (void );extern void mce_self_interrupt (void ); +extern void nmi_delayed_call_interrupt (void );extern void invalidate_interrupt (void );extern void invalidate_interrupt0 (void );
arch/x86/include/asm/irq_vectors.h +5 -0
@@ -125,6 +125,11 @@*/ #define MCE_SELF_VECTOR 0xeb +/* +* Self IPI vector for NMI delayed call +*/ +#define NMI_DELAYED_CALL_VECTOR 0xe9 +#define NR_VECTORS 256#define FPU_IRQ 13
arch/x86/include/asm/nmi.h +7 -0
@@ -75,4 +75,11 @@ void enable_lapic_nmi_watchdog(void);void stop_nmi (void );void restart_nmi (void ); +#define NMI_DELAYED_CALL_ID_INVALID -1 + +typedef void (*nmi_delayed_call_func_t )(void ); +int nmi_delayed_call_register (nmi_delayed_call_func_t func); +void nmi_delayed_call_unregister (int id); +void nmi_delayed_call_schedule (int id); +#endif /* _ASM_X86_NMI_H */
arch/x86/kernel/entry_64.S +3 -0
@@ -1009,6 +1009,9 @@ apicinterrupt MCE_SELF_VECTOR \ mce_self_interrupt smp_mce_self_interrupt #endif +apicinterrupt NMI_DELAYED_CALL_VECTOR \ + nmi_delayed_call_interrupt smp_nmi_delayed_call_interrupt + #ifdef CONFIG_SMP apicinterrupt CALL_FUNCTION_SINGLE_VECTOR \ call_function_single_interrupt smp_call_function_single_interrupt
arch/x86/kernel/irqinit.c +3 -0
@@ -212,6 +212,9 @@ static void __init apic_intr_init(void)#if defined(CONFIG_X86_MCE ) && defined(CONFIG_X86_LOCAL_APIC )alloc_intr_gate (MCE_SELF_VECTOR , mce_self_interrupt);#endif +#if defined(CONFIG_X86_LOCAL_APIC ) +alloc_intr_gate (NMI_DELAYED_CALL_VECTOR , nmi_delayed_call_interrupt); +#endif #if defined(CONFIG_X86_64 ) || defined(CONFIG_X86_LOCAL_APIC )/* self generated IPI for local APIC timer */
arch/x86/kernel/traps.c +104 -0
107 lines not on this page · raw message
RFC 2/3 Use unified NMI delayed call mechanism in MCE handler
The original self interrupt mechanism in MCE handler is replaced by the unified delayed call mechanism.
--- arch/x86/include/asm/entry_arch.h | 4 -- arch/x86/include/asm/irq_vectors.h | 5 --- arch/x86/kernel/cpu/mcheck/mce.c | 53 ++++++------------------------------- arch/x86/kernel/entry_64.S | 5 --- arch/x86/kernel/irqinit.c | 3 -- 5 files changed, 10 insertions(+), 60 deletions(-)
arch/x86/include/asm/entry_arch.h +0 -4
@@ -61,9 +61,5 @@ BUILD_INTERRUPT(thermal_interrupt,THERMA BUILD_INTERRUPT(threshold_interrupt,THRESHOLD_APIC_VECTOR)#endif -#ifdef CONFIG_X86_MCE -BUILD_INTERRUPT(mce_self_interrupt,MCE_SELF_VECTOR) -#endif - BUILD_INTERRUPT(nmi_delayed_call_interrupt,NMI_DELAYED_CALL_VECTOR)#endif
arch/x86/include/asm/irq_vectors.h +0 -5
@@ -121,11 +121,6 @@#define UV_BAU_MESSAGE 0xea/* -* Self IPI vector for machine checks -*/ -#define MCE_SELF_VECTOR 0xeb - -/* * Self IPI vector for NMI delayed call */ #define NMI_DELAYED_CALL_VECTOR 0xe9
arch/x86/kernel/cpu/mcheck/mce.c +10 -43
76 lines not on this page · raw message
arch/x86/kernel/entry_64.S +0 -5
@@ -1004,11 +1004,6 @@ apicinterrupt THRESHOLD_APIC_VECTOR \ apicinterrupt THERMAL_APIC_VECTOR \ thermal_interrupt smp_thermal_interrupt -#ifdef CONFIG_X86_MCE -apicinterrupt MCE_SELF_VECTOR \ - mce_self_interrupt smp_mce_self_interrupt -#endif - apicinterrupt NMI_DELAYED_CALL_VECTOR \ nmi_delayed_call_interrupt smp_nmi_delayed_call_interrupt
arch/x86/kernel/irqinit.c +0 -3
@@ -209,9 +209,6 @@ static void __init apic_intr_init(void)#ifdef CONFIG_X86_MCE_THRESHOLD alloc_intr_gate (THRESHOLD_APIC_VECTOR , threshold_interrupt);#endif -#if defined(CONFIG_X86_MCE ) && defined(CONFIG_X86_LOCAL_APIC ) -alloc_intr_gate (MCE_SELF_VECTOR , mce_self_interrupt); -#endif #if defined(CONFIG_X86_LOCAL_APIC )alloc_intr_gate (NMI_DELAYED_CALL_VECTOR , nmi_delayed_call_interrupt);#endif
RFC 3/3 Use unified NMI delayed call mechanism in perf event NMI handler
In reply to Huang YingThe original self interrupt mechanism in perf event NMI handler is replaced by the unified delayed call mechanism.
--- arch/x86/include/asm/entry_arch.h | 4 ---- arch/x86/include/asm/irq_vectors.h | 5 ----- arch/x86/kernel/cpu/perf_event.c | 13 ++++++++----- arch/x86/kernel/entry_64.S | 5 ----- arch/x86/kernel/irqinit.c | 6 ------ 5 files changed, 8 insertions(+), 25 deletions(-)
arch/x86/include/asm/entry_arch.h +0 -4
@@ -49,10 +49,6 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOC BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) -#ifdef CONFIG_PERF_EVENTS -BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR) -#endif -#ifdef CONFIG_X86_THERMAL_VECTOR BUILD_INTERRUPT(thermal_interrupt,THERMAL_APIC_VECTOR)#endif
arch/x86/include/asm/irq_vectors.h +0 -5
@@ -113,11 +113,6 @@*/ #define X86_PLATFORM_IPI_VECTOR 0xed -/* -* Performance monitoring pending work vector: -*/ -#define LOCAL_PENDING_VECTOR 0xec -#define UV_BAU_MESSAGE 0xea/*
arch/x86/kernel/cpu/perf_event.c +8 -5
@@ -1160,13 +1160,12 @@ static int x86_pmu_handle_irq(struct pt_return handled; } -void smp_perf_pending_interrupt (struct pt_regs *regs) +static int perf_delayed_call_id =NMI_DELAYED_CALL_ID_INVALID ; + +void perf_do_pending (void ) { -irq_enter (); -ack_APIC_irq ();inc_irq_stat (apic_pending_irqs);perf_event_do_pending (); -irq_exit (); }void set_perf_event_pending (void )
@@ -1175,7 +1174,7 @@ void set_perf_event_pending(void)if (!x86_pmu.apic || !x86_pmu_initialized ())return ; - apic->send_IPI_self (LOCAL_PENDING_VECTOR ); +nmi_delayed_call_schedule (perf_delayed_call_id);#endif }
@@ -1363,6 +1362,10 @@ void __init init_hw_perf_events(void) } } + perf_delayed_call_id =nmi_delayed_call_register (perf_do_pending); +if (perf_delayed_call_id ==NMI_DELAYED_CALL_ID_INVALID ) +pr_err ("Failed to register NMI delayed call for perf.\n" ); +pr_info ("... version: %d\n" , x86_pmu.version);pr_info ("... bit width: %d\n" , x86_pmu.cntval_bits);pr_info ("... generic registers: %d\n" , x86_pmu.num_counters);
arch/x86/kernel/entry_64.S +0 -5
@@ -1021,11 +1021,6 @@ apicinterrupt ERROR_APIC_VECTOR \ apicinterrupt SPURIOUS_APIC_VECTOR \ spurious_interrupt smp_spurious_interrupt -#ifdef CONFIG_PERF_EVENTS -apicinterrupt LOCAL_PENDING_VECTOR \ - perf_pending_interrupt smp_perf_pending_interrupt -#endif - /* * Exception entry points. */
arch/x86/kernel/irqinit.c +0 -6
@@ -223,12 +223,6 @@ static void __init apic_intr_init(void)/* IPI vectors for APIC spurious and error interrupts */ alloc_intr_gate (SPURIOUS_APIC_VECTOR , spurious_interrupt);alloc_intr_gate (ERROR_APIC_VECTOR , error_interrupt); - -/* Performance monitoring interrupts: */ -# ifdef CONFIG_PERF_EVENTS -alloc_intr_gate (LOCAL_PENDING_VECTOR , perf_pending_interrupt); -# endif -#endif }
Quoted 8 lines · Huang Ying
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
Signed-off-by Huang Ying <[email protected]>
Quoted 9 lines
--- arch/x86/include/asm/entry_arch.h | 1 arch/x86/include/asm/hw_irq.h | 1 arch/x86/include/asm/irq_vectors.h | 5 + arch/x86/include/asm/nmi.h | 7 ++ arch/x86/kernel/entry_64.S | 3 + arch/x86/kernel/irqinit.c | 3 + arch/x86/kernel/traps.c | 104 +++++++++++++++++++++++++++++++++++++ 7 files changed, 124 insertions(+)
Instead of introducing this extra intermediate facility please use the same approach the unified NMI watchdog is using (see latest -tip): a perf event callback gives all the extra functionality needed.
The MCE code needs to be updated to use that - and then it will be integrated into the events framework.
Thanks,
Ingo
Quoted 20 lines · Ingo Molnar
Quoted 8 lines · Huang Ying
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
Signed-off-by Huang Ying <[email protected]>Quoted 9 lines
--- arch/x86/include/asm/entry_arch.h | 1 arch/x86/include/asm/hw_irq.h | 1 arch/x86/include/asm/irq_vectors.h | 5 + arch/x86/include/asm/nmi.h | 7 ++ arch/x86/kernel/entry_64.S | 3 + arch/x86/kernel/irqinit.c | 3 + arch/x86/kernel/traps.c | 104 +++++++++++++++++++++++++++++++++++++ 7 files changed, 124 insertions(+)
Quoted 3 lines
Instead of introducing this extra intermediate facility please use the same approach the unified NMI watchdog is using (see latest -tip): a perf event callback gives all the extra functionality needed.
Sorry, if my understanding is correct, the perf event overflow callback should be run in NMI context instead of a delayed context (such as IRQ, soft_irq, process context). That is, the backtrace of watchdog_overflow_callback should be something as follow:
x86_pmu_handle_irq
perf_event_overflow
__perf_event_overflow
watchdog_overflow_callback
I do not find the delayed mechanism here.
Quoted 2 lines
The MCE code needs to be updated to use that - and then it will be integrated into the events framework.
MCE is NMI-like, and there are other NMI users too. I think some of them will need some kind of delayed call mechanism. In fact, perf itself uses self-made NMI delayed call mechanism too, I just want to generalize it for other users too.
Best Regards,
Huang Ying
Quoted 10 lines · Ingo Molnar
Quoted 6 lines · Huang Ying
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
Quoted 6 lines
Instead of introducing this extra intermediate facility please use the same approach the unified NMI watchdog is using (see latest -tip): a perf event callback gives all the extra functionality needed.
The MCE code needs to be updated to use that - and then it will be integrated into the events framework.
Hi Ingo,
I think this "NMI delayed call mechanism" could be a part of "the events framework" that we are planning to get in kernel soon. At least APEI will use NMI to report some hardware events (likely error) to kernel. So I suppose we will go to have a delayed call as an event handler for APEI.
Generally speaking "event" can occur independently of the situation. NMI can tell us some of external events, expecting urgent reaction for the event, but we cannot do everything in NMI context. Or we might have a sudden urge to generate an internal event while interrupts are disabled.
I agree that generating a self interrupt is reasonable solution. Note that it could be said that both of "MCE handled (=event log should be delivered to userland asap)" and "perf events pending (=pending events should be handled asap)" are kind of internal event that requires urgent handling in non-NMI kernel context. One question here is why we should have different vectors for these events that uses same mechanism.
How about calling the vector LOCAL_EVENT_VECTOR or so?
I guess there should be better name if it is possible to inject an event to other cpus via IPI with this vector...
Thanks,
H.Seto
Quoted 30 lines · Hidetoshi Seto
Quoted 10 lines · Ingo Molnar
Quoted 6 lines · Huang Ying
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
Quoted 6 lines
Instead of introducing this extra intermediate facility please use the same approach the unified NMI watchdog is using (see latest -tip): a perf event callback gives all the extra functionality needed.
The MCE code needs to be updated to use that - and then it will be integrated into the events framework.
Hi Ingo,
I think this "NMI delayed call mechanism" could be a part of "the events framework" that we are planning to get in kernel soon. At least APEI will use NMI to report some hardware events (likely error) to kernel. So I suppose we will go to have a delayed call as an event handler for APEI.
Generally speaking "event" can occur independently of the situation. NMI can tell us some of external events, expecting urgent reaction for the event, but we cannot do everything in NMI context. Or we might have a sudden urge to generate an internal event while interrupts are disabled.
Quoted 6 lines
I agree that generating a self interrupt is reasonable solution. Note that it could be said that both of "MCE handled (=event log should be delivered to userland asap)" and "perf events pending (=pending events should be handled asap)" are kind of internal event that requires urgent handling in non-NMI kernel context. One question here is why we should have different vectors for these events that uses same mechanism.
I think the perf event subsytem can log events in NMI context already and deliver them to userspace when the NMI is done. This is why I think Ingo wants MCE to be updated to sit on top of the perf event subsytem to avoid re-invent everything again.
Then again I do not know enough about the MCE stuff to understand what you mean when an event comes in but you can't handle it in an NMI-safe context. An example would be helpful.
Cheers,
Don
Quoted 4 lines
I think the perf event subsytem can log events in NMI context already and deliver them to userspace when the NMI is done. This is why I think Ingo wants MCE to be updated to sit on top of the perf event subsytem to avoid re-invent everything again.
perf is not solving the problem this is trying to solve.
Quoted 3 lines
Then again I do not know enough about the MCE stuff to understand what you mean when an event comes in but you can't handle it in an NMI-safe context. An example would be helpful.
At least for MCE hwpoison recovery needs to sleep and you obviously cannot sleep in NMI like context. The way it's done is to first do a self interrupt, then do a work queue wakeup and finally the sleeping operations.
perf does not fit into this because it has no way to process such an event inside the kernel.
Anyways this just cleans up the existing mechanism to share some code.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Quoted 11 lines · Andi Kleen
Quoted 4 lines
I think the perf event subsytem can log events in NMI context already and deliver them to userspace when the NMI is done. This is why I think Ingo wants MCE to be updated to sit on top of the perf event subsytem to avoid re-invent everything again.
perf is not solving the problem this is trying to solve.
Quoted 3 lines
Then again I do not know enough about the MCE stuff to understand what you mean when an event comes in but you can't handle it in an NMI-safe context. An example would be helpful.
Quoted 6 lines
At least for MCE hwpoison recovery needs to sleep and you obviously cannot sleep in NMI like context. The way it's done is to first do a self interrupt, then do a work queue wakeup and finally the sleeping operations.
perf does not fit into this because it has no way to process such an event inside the kernel.
Ah, makes sense. Thanks for the example.
Cheers,
Don
Quoted 6 lines · Andi Kleen
Quoted 4 lines
I think the perf event subsytem can log events in NMI context already and deliver them to userspace when the NMI is done. This is why I think Ingo wants MCE to be updated to sit on top of the perf event subsytem to avoid re-invent everything again.
perf is not solving the problem this is trying to solve.
That is why i requested to extend the events backend. That will unify more of the code than the first few steps achieved by these three patches - and offers the functionality to all code that uses the events framework.
Quoted 4 lines
[...]
perf does not fit into this because it has no way to process such an event inside the kernel.
It 'does not fit' into the events backend only if you pretend that it is impossible or undesirable to have a delayed, in-context callback mechanism implemented there.
If you look at it more closely you'll notice that in reality it's not only possible but that it is also a pretty natural fit.
Thanks,
Ingo
Quoted 15 lines · Hidetoshi Seto
Quoted 14 lines · Ingo Molnar
Quoted 6 lines · Huang Ying
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
Instead of introducing this extra intermediate facility please use the same approach the unified NMI watchdog is using (see latest -tip): a perf event callback gives all the extra functionality needed.
Quoted 7 lines
Quoted 2 lines
The MCE code needs to be updated to use that - and then it will be integrated into the events framework.
Hi Ingo,
I think this "NMI delayed call mechanism" could be a part of "the events framework" that we are planning to get in kernel soon. [...]
My request was to make it part of perf events - which is a generic event logging framework. We dont really need/want a second 'events framework' as we have one already ;-)
Quoted 3 lines
[...] At least APEI will use NMI to report some hardware events (likely error) to kernel. So I suppose we will go to have a delayed call as an event handler for APEI.
Yep, that makes sense. I wasnt arguing against the functionality itself, i was arguing against the illogical layering that limits its utility. By making it part of perf events it becomes a generic part of that framework and can be used by anything that deals with events and uses that framework.
Thanks,
Ingo
Hi, Ingo,
Quoted 21 lines · Ingo Molnar
Quoted 18 lines · Hidetoshi Seto
Quoted 16 lines · Ingo Molnar
Quoted 6 lines · Huang Ying
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
Instead of introducing this extra intermediate facility please use the same approach the unified NMI watchdog is using (see latest -tip): a perf event callback gives all the extra functionality needed.
The MCE code needs to be updated to use that - and then it will be integrated into the events framework.
Quoted 8 lines
Quoted 4 lines
Hi Ingo,
I think this "NMI delayed call mechanism" could be a part of "the events framework" that we are planning to get in kernel soon. [...]
My request was to make it part of perf events - which is a generic event logging framework. We dont really need/want a second 'events framework' as we have one already ;-)
This patchset is simple and straightforward, it is just a delayed execution mechanism, not another 'events framework'. There are several other NMI users other than perf, should we integrate all NMI users into perf framework?
Quoted 8 lines
Quoted 3 lines
[...] At least APEI will use NMI to report some hardware events (likely error) to kernel. So I suppose we will go to have a delayed call as an event handler for APEI.
Yep, that makes sense. I wasnt arguing against the functionality itself, i was arguing against the illogical layering that limits its utility. By making it part of perf events it becomes a generic part of that framework and can be used by anything that deals with events and uses that framework.
I think the the 'layering' in the patchset helps instead of 'limits' its utility. It is designed to be as general as possible, so that it can be used by both perf and other NMI users. Do you think so?
Best Regards,
Huang Ying
Quoted 26 lines · huang ying
Hi, Ingo,
Quoted 23 lines · Ingo Molnar
Quoted 20 lines · Hidetoshi Seto
Quoted 10 lines · Ingo Molnar
Quoted 6 lines · Huang Ying
NMI can be triggered even when IRQ is masked. So it is not safe for NMI handler to call some functions. One solution is to delay the call via self interrupt, so that the delayed call can be done once the interrupt is enabled again. This has been implemented in MCE and perf event. This patch provides a unified version and make it easier for other NMI semantic handler to take use of the delayed call.
Quoted 6 lines
Instead of introducing this extra intermediate facility please use the same approach the unified NMI watchdog is using (see latest -tip): a perf event callback gives all the extra functionality needed.
The MCE code needs to be updated to use that - and then it will be integrated into the events framework.
Hi Ingo,
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
I think this "NMI delayed call mechanism" could be a part of "the events framework" that we are planning to get in kernel soon. [...]
My request was to make it part of perf events - which is a generic event logging framework. We dont really need/want a second 'events framework' as we have one already ;-)
This patchset is simple and straightforward, [...]
We wouldnt want to add another workqueue or memory allocation mechanism either, even if it was 'simple and straightforward'. We try to make things more generally useful.
Quoted 3 lines
[...] it is just a delayed execution mechanism, not another 'events framework'. There are several other NMI users other than perf, should we integrate all NMI users into perf framework?
We already did so with the NMI watchdog. What other significant NMI event users do you have in mind?
Quoted 10 lines
Quoted 9 lines
Quoted 3 lines
[...] ??At least APEI will use NMI to report some hardware events (likely error) to kernel. ??So I suppose we will go to have a delayed call as an event handler for APEI.
Yep, that makes sense. I wasnt arguing against the functionality itself, i was arguing against the illogical layering that limits its utility. By making it part of perf events it becomes a generic part of that framework and can be used by anything that deals with events and uses that framework.
Quoted 3 lines
I think the the 'layering' in the patchset helps instead of 'limits' its utility. It is designed to be as general as possible, so that it can be used by both perf and other NMI users. Do you think so?
What other NMI users do you mean? EDAC/MCE is going to go utilize events as well (away from the horrible /dev/mcelog interface), the NMI watchdog already did it and the perf tool obviously does as well. There's a few leftovers like kcrash which isnt really event centric and i dont think it needs to be converted.
Thanks,
Ingo
Quoted 14 lines · Ingo Molnar
Quoted 10 lines
Quoted 9 lines
Quoted 3 lines
[...] ??At least APEI will use NMI to report some hardware events (likely error) to kernel. ??So I suppose we will go to have a delayed call as an event handler for APEI.
Yep, that makes sense. I wasnt arguing against the functionality itself, i was arguing against the illogical layering that limits its utility. By making it part of perf events it becomes a generic part of that framework and can be used by anything that deals with events and uses that framework.
Quoted 3 lines
I think the the 'layering' in the patchset helps instead of 'limits' its utility. It is designed to be as general as possible, so that it can be used by both perf and other NMI users. Do you think so?
Quoted 5 lines
What other NMI users do you mean? EDAC/MCE is going to go utilize events as well (away from the horrible
/dev/mceloginterface), the NMI watchdog already did it and the perf tool obviously does as well. There's a few leftovers like kcrash which isnt really event centric and i dont think it needs to be converted.
But why not just make it more general? It does not hurt anyone including perf.
Best Regards,
Huang Ying
Quoted 15 lines · huang ying
Quoted 14 lines · Ingo Molnar
Quoted 13 lines
Quoted 9 lines
Quoted 3 lines
[...] ??At least APEI will use NMI to report some hardware events (likely error) to kernel. ??So I suppose we will go to have a delayed call as an event handler for APEI.
Yep, that makes sense. I wasnt arguing against the functionality itself, i was arguing against the illogical layering that limits its utility. By making it part of perf events it becomes a generic part of that framework and can be used by anything that deals with events and uses that framework.
I think the the 'layering' in the patchset helps instead of 'limits' its utility. It is designed to be as general as possible, so that it can be used by both perf and other NMI users. Do you think so?
Quoted 8 lines
Quoted 5 lines
What other NMI users do you mean? EDAC/MCE is going to go utilize events as well (away from the horrible
/dev/mceloginterface), the NMI watchdog already did it and the perf tool obviously does as well. There's a few leftovers like kcrash which isnt really event centric and i dont think it needs to be converted.But why not just make it more general? It does not hurt anyone including perf.
Because it's not actually more generic that way - just look at the code. It's x86 specific, plus it ties it to NMI delivery while the concept of delayed execution has nothing to do with NMIs.
Ingo
Quoted 24 lines · Ingo Molnar
Quoted 21 lines · huang ying
Quoted 19 lines · Ingo Molnar
Quoted 10 lines
Quoted 9 lines
Quoted 3 lines
[...] ??At least APEI will use NMI to report some hardware events (likely error) to kernel. ??So I suppose we will go to have a delayed call as an event handler for APEI.
Yep, that makes sense. I wasnt arguing against the functionality itself, i was arguing against the illogical layering that limits its utility. By making it part of perf events it becomes a generic part of that framework and can be used by anything that deals with events and uses that framework.
Quoted 3 lines
I think the the 'layering' in the patchset helps instead of 'limits' its utility. It is designed to be as general as possible, so that it can be used by both perf and other NMI users. Do you think so?
What other NMI users do you mean? EDAC/MCE is going to go utilize events as well (away from the horrible
/dev/mceloginterface), the NMI watchdog already did it and the perf tool obviously does as well. There's a few leftovers like kcrash which isnt really event centric and i dont think it needs to be converted.
Quoted 6 lines
Quoted 2 lines
But why not just make it more general? It does not hurt anyone including perf.
Because it's not actually more generic that way - just look at the code. It's x86 specific, plus it ties it to NMI delivery while the concept of delayed execution has nothing to do with NMIs.
soft_irq is a delayed mechanism for IRQ, a self interrupt can be a delayed mechanism for NMI. If we can make soft_irq NMI-safe, we can use soft_irq as a backup of self interrupt (for systems without APIC and maybe for other architectures).
Best Regards,
Huang Ying
Peter Zijlstra
maintainerQuoted 2 lines · huang ying
soft_irqis a delayed mechanism for IRQ,
No its not.
Quoted 2 lines
a self interrupt can be a
delayed mechanism for NMI. If we can makesoft_irqNMI-safe,
No you can't.
Quoted 3 lines
we can
usesoft_irqas a backup of self interrupt (for systems without APIC and maybe for other architectures).
Whatever would you want to do that for.
Quoted 5 lines · Peter Zijlstra
Quoted 2 lines · huang ying
soft_irqis a delayed mechanism for IRQ,No its not.
Why? What do you think soft_irq is for?
Quoted 10 lines
Quoted 2 lines
a self interrupt can be a
delayed mechanism for NMI. If we can makesoft_irqNMI-safe,No you can't.
Quoted 3 lines
we can
usesoft_irqas a backup of self interrupt (for systems without APIC and maybe for other architectures).Whatever would you want to do that for.
Best Regards,
Huang Ying
Quoted 5 lines
Quoted 3 lines
we can
usesoft_irqas a backup of self interrupt (for systems without APIC and maybe for other architectures).Whatever would you want to do that for.
The idea is that the work would be done latest on the next timer interrupt as a fallback if APIC is not available.
That's what mce does already and it's probably approbiate for most other users too.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
( Ugh: what's this new fad of you not quoting the name of the person who
wrote a mail? It makes multi-level quotes utterly unreadable as it's not clear who wrote what. You should also respect others by quoting their names. )
Quoted 7 lines · Andi Kleen
Quoted 4 lines
Quoted 2 lines
we can use
soft_irqas a backup of self interrupt (for systems without APIC and maybe for other architectures).Whatever would you want to do that for.
The idea is that the work would be done latest on the next timer interrupt as a fallback if APIC is not available.
Abusing the timer irq for that is an exceedingly ugly and unacceptable design, as machine check events have nothing to do with timers. (That approach is also buggy because it inserts an arbitrary delay - which could be rather long on nohz.)
This kind of messy, ad-hoc piggybacking only creates unmaintainable code in the long run.
Quoted 2 lines
That's what mce does already and it's probably approbiate for most other users too.
Hell no, the unfortunate and unclean practices of the MCE code must not be propagated elsewhere.
The proper, generic approach would be to enable softirq notifications (on x86) from NMI contexts as well (it's actually possible without overhead), and to extend user return notifiers with the logical next step: nmi return notifiers. If presented in such a form then those could use softirqs for atomic callbacks and per cpu kthreads for sleepable callbacks, etc.
Ingo
Quoted 2 lines · Ingo Molnar
The proper, generic approach would be to enable softirq notifications (on x86) from NMI contexts as well (it's actually possible without overhead),
Yes. I will do that. And I think self interrupt can be used as the short-cut for soft_irq if available. The next soft_irq may be too late if there is too few interrupts.
Quoted 4 lines
and to
extend user return notifiers with the logical next step: nmi return notifiers. If presented in such a form then those could use softirqs for atomic callbacks and per cpu kthreads for sleepable callbacks, etc.
NMI return notifiers fired in soft_irq?
Best Regards,
Huang Ying
Quoted 4 lines · Ingo Molnar
( Ugh: what's this new fad of you not quoting the name of the person who
wrote a mail? It makes multi-level quotes utterly unreadable as it's not clear who wrote what. You should also respect others by quoting their names. )
To give you an excuse to flame of course. I know that's your favourite pastime and I'm always happy to make you happy.
Quoted 3 lines
Abusing the timer irq for that is an exceedingly ugly and unacceptable design, as machine check events have nothing to do with timers. (That approach is also
The rationale is that this nearly never happens in practice anyways (the operations that trigger it do near always only happen with APICs) AFAIK that's true for all the proposed users.
For very rare and obscure cases like this in my experience the simplest possible code is the best,
that makes it most likely it actually works when it's needed. I do not know of any simpler way to implement this.
Of course if there's evidence it's actually common that would need to be revisited.
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Peter Zijlstra
maintainerQuoted 20 lines · Huang Ying
arch/x86/kernel/traps.c +20
-0+#define NMI_DELAYED_CALL_ID_MAX 32 +#define NMI_DELAYED_CALL_RESTART_MAX 5 + +static nmi_delayed_call_func_t nmi_delayed_call_funcs[NMI_DELAYED_CALL_ID_MAX ]; +static DEFINE_SPINLOCK (nmi_delayed_call_lock); + +static DEFINE_PER_CPU(unsigned long, nmi_delayed_call_pending); + +static void nmi_delayed_call_run (void ) +{ +int cpu, restart =NMI_DELAYED_CALL_RESTART_MAX ; +unsigned long pending, *ppending; +nmi_delayed_call_func_t *pfunc, func; + + cpu =smp_processor_id (); + ppending =per_cpu_ptr (&nmi_delayed_call_pending, cpu); +while (*ppending && restart--) { + pending =xchg (ppending,0 ); + pfunc = nmi_delayed_call_funcs; + do {
arch/x86/kernel/traps.c void __init trap_init(void) · line 921
Quoted 10 lines
arch/x86/kernel/traps.c +10
-0+if (pending &1 ) { + func = *pfunc; +if (func) +func (); + } + pfunc++; + pending >>=1 ; + }while (pending); + } +}
So aside from the should this be perf or not, the above is utter gibberish. Whoever came up with this nonsense?
Why not make a work_struct like thing and enqueue it using cmpxchg on a percpu list, then have the interrupt process them. Read perf_pending_queue() and __perf_pending_run().
That way you don't need this whole register/id/limit crap.
Quoted 1 line
+#ifdef
CONFIG_X86_LOCAL_APIC
What's the point of the rest of this code if we don't have a lapic?
arch/x86/kernel/traps.c void __init trap_init(void) · line 927
Quoted 4 lines
arch/x86/kernel/traps.c +4
-0+asmlinkage void smp_nmi_delayed_call_interrupt(struct pt_regs *regs) +{ +ack_APIC_irq (); +irq_enter ();
You're missing inc_irq_stat() there.
Quoted 4 lines
arch/x86/kernel/traps.c +4
-0+nmi_delayed_call_run (); + irq_exit(); +} +#endif
Quoted 2 lines
So aside from the should this be perf or not, the above is utter gibberish. Whoever came up with this nonsense?
This is pretty much how softirqs (and before them bottom halves) work. I believe Linus invented that scheme originally back in the early days of Linux.
It's actually quite simple and works well
-Andi
Peter Zijlstra
maintainerQuoted 6 lines · Andi Kleen
Quoted 2 lines
So aside from the should this be perf or not, the above is utter gibberish. Whoever came up with this nonsense?
This is pretty much how softirqs (and before them bottom halves) work. I believe Linus invented that scheme originally back in the early days of Linux.
Doesn't mean its the right abstraction for this.
Quoted 1 line
It's actually quite simple and works well
And adds more code than it removes whilst providing a very limited service.
You generally want to pass more information along anyway, now your callback function needs to go look for it. Much better to pass a work_struct like thing around that is contained in the state it needs.
Quoted 3 lines
You generally want to pass more information along anyway, now your callback function needs to go look for it. Much better to pass a
work_structlike thing around that is contained in the state it needs.
But how would you allocate the work queue in an NMI?
If it's only a single instance (like this bit) it can be always put into a per cpu variable.
-Andi
Peter Zijlstra
maintainerQuoted 8 lines · Andi Kleen
Quoted 3 lines
You generally want to pass more information along anyway, now your callback function needs to go look for it. Much better to pass a
work_structlike thing around that is contained in the state it needs.But how would you allocate the work queue in an NMI?
If it's only a single instance (like this bit) it can be always put into a per cpu variable.
Pre-allocate. For the perf-event stuff we use the perf_event allocated at creation time. But yeah, per-cpu storage also works.
Quoted 5 lines · Peter Zijlstra
Quoted 4 lines · Andi Kleen
Quoted 3 lines
You generally want to pass more information along anyway, now your callback function needs to go look for it. Much better to pass a
work_structlike thing around that is contained in the state it needs.
Quoted 7 lines
Quoted 4 lines
But how would you allocate the work queue in an NMI?
If it's only a single instance (like this bit) it can be always put into a per cpu variable.
Pre-allocate. For the perf-event stuff we use the
perf_eventallocated at creation time. But yeah, per-cpu storage also works.
So you could just preallocate the bits instead ?
-Andi
Signature · 1 line
[email protected] -- Speaking for myself only.
Peter Zijlstra
maintainerQuoted 8 lines · Andi Kleen
Quoted 7 lines · Peter Zijlstra
Quoted 6 lines · Andi Kleen
Quoted 3 lines
You generally want to pass more information along anyway, now your callback function needs to go look for it. Much better to pass a
work_structlike thing around that is contained in the state it needs.But how would you allocate the work queue in an NMI?
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
If it's only a single instance (like this bit) it can be always put into a per cpu variable.
Pre-allocate. For the perf-event stuff we use the
perf_eventallocated at creation time. But yeah, per-cpu storage also works.So you could just preallocate the bits instead ?
You mean the bits in your function array? Those are limited to 32 and you'd need a secondary lookup to match them to your data object, not very useful.
Quoted 6 lines · Andi Kleen
Quoted 2 lines
So aside from the should this be perf or not, the above is utter gibberish. Whoever came up with this nonsense?
This is pretty much how softirqs (and before them bottom halves) work. I believe Linus invented that scheme originally back in the early days of Linux.
Nope - the softirq code was written by Alexey Kuznetsov and David S. Miller. But it's irrelevant, because:
Quoted 1 line
It's actually quite simple and works well
... as Peter said the last thing we want is yet another softirq vector.
Thanks,
Ingo
Quoted 1 line
Nope - the softirq code was written by Alexey Kuznetsov and David S. Miller.
It was based on the design of the previous bottom halves in case you missed it. I believe BHs were there from nearly the beginning.
(softirqs were basically just per cpu bottom halves, otherwise being very similar)
-Andi