interrupt: Disable interrupt before modifying hardirq_disable counter
PATCH 1/1 interrupt: Disable interrupt before modifying hardirq_disable counter
Currently a softirq may be pending longer then expected if the triggering interrupt happens in-between hardirq_disable_enter() and _local_interrupt_disable() in local_interrupt_disable():
local_interrupt_disable():
hardirq_disable_enter();
<interrupt>
...
__irq_exit_rcu():
// false because hardirq_disable_count() is not 0
if (.. && !hardirq_disable_count() && ..) {
invoke_softirq();
}
_local_interrupt_disable();, it'll defer the softirq to the next interrupt which can be forever.
The order between hardirq_disable_enter() and _local_interrupt_disable() is to optimize re-disabling interrupts if they are already disabled, but as 1) local_interrupt_disable() is not widely used yet and 2) the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq pending issue.
Since we are doing this, the part of saving the current state is separated from irq disabling, and we basically do the following in local_interrupt_disable():
local_irq_save(flags);
if (counter++ == 0) {
this_cpu(local_interrupt_disable_state) = flags;
}Change _local_interrupt_disable() to _local_interrupt_save_state().
--- include/linux/interrupt_rc.h | 19 ++++++++----------- kernel/softirq.c | 17 ++++------------- 2 files changed, 12 insertions(+), 24 deletions(-)
include/linux/interrupt_rc.h +8 -11
@@ -20,11 +20,8 @@/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); -static __always_inline void __local_interrupt_disable(void) +static __always_inline void __local_interrupt_save_state(unsigned long flags) { -unsigned long flags; - -local_irq_save (flags);raw_cpu_write (local_interrupt_disable_state, flags); }
@@ -36,9 +33,9 @@ static __always_inline void __local_interrupt_enable(void) }#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE -static __always_inline void _local_interrupt_disable(void) +static __always_inline void _local_interrupt_save_state(unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } static __always_inline void _local_interrupt_enable(void)
@@ -46,27 +43,27 @@ static __always_inline void _local_interrupt_enable(void)__local_interrupt_enable (); }#else -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif #else /* !MODULE */ -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif /* !MODULE */ static inline void local_interrupt_disable (void ) {int new_count; +unsigned long flags;WARN_ON_ONCE (in_nmi ()); +local_irq_save (flags); new_count =hardirq_disable_enter (); -/* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ -if ((new_count &HARDIRQ_DISABLE_MASK ) ==HARDIRQ_DISABLE_OFFSET ) -_local_interrupt_disable (); +_local_interrupt_save_state (flags); }static inline void local_interrupt_enable (void )
kernel/softirq.c +4 -13
@@ -91,11 +91,11 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); -void _local_interrupt_disable (void ) +void _local_interrupt_save_state (unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } -EXPORT_SYMBOL (_local_interrupt_disable); +EXPORT_SYMBOL (_local_interrupt_save_state);void _local_interrupt_enable (void ) {
@@ -749,16 +749,7 @@ static inline void __irq_exit_rcu(void)#endif account_hardirq_exit (current);preempt_count_sub (HARDIRQ_OFFSET ); -/* -* Interrupts may happen between hardirq_disable_enter() and -* local_irq_save() in local_interrupt_disable(), if irq_exit() invokes -* softirq here, we may have a softirq handler calling -* local_interrupt_disable() but it won't disable the IRQ because -* hardirq disabling count is already 1, hence we need to prevent -* invoking softirq when a local_interrupt_disable() is ongoing. -*/ -if (!in_interrupt () && !hardirq_disable_count () && -local_softirq_pending ()) { +if (!in_interrupt () &&local_softirq_pending ()) {/* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run softirq.
Signature · 1 line
2.50.1 (Apple Git-155)
PATCH 1/1 preempt: Remove hardirq_disable_count()
It turns out the previous usage of hardirq_disable_count() in __irq_exit_rcu() would cause softirq pending issues. Without that usage, hardirq_disable_count() doesn't need to exist, so remove it.
--- include/linux/preempt.h | 1 - 1 file changed, 1 deletion(-)
include/linux/preempt.h +0 -1
@@ -168,7 +168,6 @@ static __always_inline unsigned char interrupt_context_level(void)#define in_softirq () (softirq_count())#define in_interrupt () (irq_count()) -#define hardirq_disable_count () ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT)#define hardirq_disable_enter () __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET)#define hardirq_disable_exit () __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET)
Signature · 1 line
2.50.1 (Apple Git-155)
[tip: locking/urgent] preempt: Remove hardirq_disable_count()
The following commit has been merged into the locking/urgent branch of tip:
The following commit has been merged into the locking/urgent branch of tip:
Commit-ID: 95a5037ec202adb579f532afb6b35680001b564e Gitweb: https://git.kernel.org/tip/95a5037ec202adb579f532afb6b35680001b564e Author: Boqun Feng <[email protected]> AuthorDate: Thu, 27 Aug 2026 12:48:35 -07:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Sun, 30 Aug 2026 08:39:04 +02:00
preempt: Remove hardirq_disable_count()
It turns out the previous usage of hardirq_disable_count() in __irq_exit_rcu() would cause softirq pending issues. Without that usage, hardirq_disable_count() doesn't need to exist, so remove it.
--- include/linux/preempt.h | 1 - 1 file changed, 1 deletion(-)
include/linux/preempt.h +0 -1
@@ -168,7 +168,6 @@ static __always_inline unsigned char interrupt_context_level(void)#define in_softirq () (softirq_count())#define in_interrupt () (irq_count()) -#define hardirq_disable_count () ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT)#define hardirq_disable_enter () __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET)#define hardirq_disable_exit () __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET)
[tip: locking/urgent] preempt: Remove hardirq_disable_count()
In reply to Boqun FengThe following commit has been merged into the locking/urgent branch of tip:
The following commit has been merged into the locking/urgent branch of tip:
Commit-ID: 2af470916a208b576ac9975d221d9a378cf8ace9 Gitweb: https://git.kernel.org/tip/2af470916a208b576ac9975d221d9a378cf8ace9 Author: Boqun Feng <[email protected]> AuthorDate: Thu, 27 Aug 2026 12:48:35 -07:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Sun, 30 Aug 2026 20:25:21 +02:00
preempt: Remove hardirq_disable_count()
It turns out the previous usage of hardirq_disable_count() in __irq_exit_rcu() would cause softirq pending issues. Without that usage, hardirq_disable_count() doesn't need to exist, so remove it.
Also move hardirq_disable_enter/exit() into the Rust specific interrupt_rc header.
[ tglx: Move the helpers over ]
--- include/linux/interrupt_rc.h | 3 +++ include/linux/preempt.h | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-)
include/linux/interrupt_rc.h +3 -0
@@ -52,6 +52,9 @@ extern void _local_interrupt_save_state(unsigned long flags);extern void _local_interrupt_enable (void );#endif /* !MODULE */ +#define hardirq_disable_enter () __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) +#define hardirq_disable_exit () __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) +static inline void local_interrupt_disable (void ) {int new_count;
include/linux/preempt.h +0 -4
@@ -168,10 +168,6 @@ static __always_inline unsigned char interrupt_context_level(void)#define in_softirq () (softirq_count())#define in_interrupt () (irq_count()) -#define hardirq_disable_count () ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT) -#define hardirq_disable_enter () __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) -#define hardirq_disable_exit () __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) -/* * The preempt_count offset after preempt_disable(); */
Quoted 33 lines · Boqun Feng
Currently a softirq may be pending longer then expected if the triggering interrupt happens in-between
hardirq_disable_enter()and_local_interrupt_disable()inlocal_interrupt_disable():local_interrupt_disable(): hardirq_disable_enter(); <interrupt> ... __irq_exit_rcu(): // false because hardirq_disable_count() is not 0 if (.. && !hardirq_disable_count() && ..) { invoke_softirq(); } _local_interrupt_disable();, it'll defer the softirq to the next interrupt which can be forever.
The order between
hardirq_disable_enter()and_local_interrupt_disable()is to optimize re-disabling interrupts if they are already disabled, but as 1)local_interrupt_disable()is not widely used yet and 2) the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq pending issue.Since we are doing this, the part of saving the current state is separated from irq disabling, and we basically do the following in
local_interrupt_disable():local_irq_save(flags); if (counter++ == 0) { this_cpu(local_interrupt_disable_state) = flags; }
Quoted 5 lines
Change
_local_interrupt_disable()to_local_interrupt_save_state().Reported-by Thomas Gleixner <[email protected]>Fixes e901c1510e24 ("irq,spin_lock: Add counted interrupt disabling/enabling")
LGTM, thanks
Quoted 97 lines
Signed-off-by Boqun Feng <[email protected]>---
include/linux/interrupt_rc.h| 19 ++++++++-----------kernel/softirq.c| 17 ++++-------------2 files changed, 12 insertions(+), 24 deletions(-)include/linux/interrupt_rc.h +0
-0@@ -20,11 +20,8 @@/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);include/linux/interrupt_rc.h +1
-1-static __always_inline void __local_interrupt_disable(void) +static __always_inline void __local_interrupt_save_state(unsigned long flags){
include/linux/interrupt_rc.h +0
-3-unsigned long flags; - -local_irq_save (flags);raw_cpu_write(local_interrupt_disable_state, flags); }include/linux/interrupt_rc.h +7
-7@@ -36,9 +33,9 @@ static __always_inline void __local_interrupt_enable(void) }#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE -static __always_inline void _local_interrupt_disable(void) +static __always_inline void _local_interrupt_save_state(unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } static __always_inline void _local_interrupt_enable(void)@@ -46,27 +43,27 @@ static __always_inline void _local_interrupt_enable(void)__local_interrupt_enable (); }#else -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif #else /* !MODULE */ -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif /* !MODULE */ static inline void local_interrupt_disable (void ) {int new_count; +unsigned long flags;WARN_ON_ONCE (in_nmi ()); +local_irq_save (flags); new_count =hardirq_disable_enter (); -/* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ -if ((new_count &HARDIRQ_DISABLE_MASK ) ==HARDIRQ_DISABLE_OFFSET ) -_local_interrupt_disable (); +_local_interrupt_save_state (flags); }static inline void local_interrupt_enable (void )kernel/softirq.c +3
-7@@ -91,11 +91,11 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); -void _local_interrupt_disable (void ) +void _local_interrupt_save_state (unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } -EXPORT_SYMBOL (_local_interrupt_disable); +EXPORT_SYMBOL (_local_interrupt_save_state);void _local_interrupt_enable (void ) {@@ -749,16 +749,7 @@ static inline void __irq_exit_rcu(void)#endif account_hardirq_exit (current);preempt_count_sub (HARDIRQ_OFFSET ); -/* -* Interrupts may happen between hardirq_disable_enter() and -* local_irq_save() in local_interrupt_disable(), if irq_exit() invokes -* softirq here, we may have a softirq handler calling
kernel/softirq.c static inline void __irq_exit_rcu(void) · line 755
Quoted 10 lines
kernel/softirq.c +1
-6-* local_interrupt_disable() but it won't disable the IRQ because -* hardirq disabling count is already 1, hence we need to prevent -* invoking softirq when a local_interrupt_disable() is ongoing. -*/ -if (!in_interrupt () && !hardirq_disable_count () && -local_softirq_pending ()) { +if (!in_interrupt () &&local_softirq_pending ()) {/* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run softirq.
Quoted 24 lines · Boqun Feng
Currently a softirq may be pending longer then expected if the triggering interrupt happens in-between
hardirq_disable_enter()and_local_interrupt_disable()inlocal_interrupt_disable():local_interrupt_disable(): hardirq_disable_enter(); <interrupt> ... __irq_exit_rcu(): // false because hardirq_disable_count() is not 0 if (.. && !hardirq_disable_count() && ..) { invoke_softirq(); } _local_interrupt_disable();, it'll defer the softirq to the next interrupt which can be forever.
The order between
hardirq_disable_enter()and_local_interrupt_disable()is to optimize re-disabling interrupts if they are already disabled, but as 1)local_interrupt_disable()is not widely used yet and 2) the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq pending issue.
Quoted 3 lines
Since we are doing this, the part of saving the current state is separated from irq disabling, and we basically do the following in
local_interrupt_disable():
We do nothing. See
https://docs.kernel.org/process/maintainer-tip.html#changelog
Quoted 8 lines
local_irq_save(flags); if (counter++ == 0) { this_cpu(local_interrupt_disable_state) = flags; }Change
_local_interrupt_disable()to_local_interrupt_save_state().
https://lore.kernel.org/lkml/87jypbfu1t.ffs@fw13/
is the mail where I explained that softirq issue.
Quoted 25 lines · Thomas Gleixner
Quoted 24 lines · Boqun Feng
Currently a softirq may be pending longer then expected if the triggering interrupt happens in-between
hardirq_disable_enter()and_local_interrupt_disable()inlocal_interrupt_disable():local_interrupt_disable(): hardirq_disable_enter(); <interrupt> ... __irq_exit_rcu(): // false because hardirq_disable_count() is not 0 if (.. && !hardirq_disable_count() && ..) { invoke_softirq(); } _local_interrupt_disable();, it'll defer the softirq to the next interrupt which can be forever.
The order between
hardirq_disable_enter()and_local_interrupt_disable()is to optimize re-disabling interrupts if they are already disabled, but as 1)local_interrupt_disable()is not widely used yet and 2) the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq pending issue.
Quoted 8 lines
Quoted 3 lines
Since we are doing this, the part of saving the current state is separated from irq disabling, and we basically do the following in
local_interrupt_disable():We do nothing. See
https://docs.kernel.org/process/maintainer-tip.html#changelog
Will fix this.
Quoted 5 lines
Quoted 5 lines
local_irq_save(flags); if (counter++ == 0) { this_cpu(local_interrupt_disable_state) = flags; }
Quoted 8 lines
Quoted 3 lines
Change
_local_interrupt_disable()to_local_interrupt_save_state().https://lore.kernel.org/lkml/87jypbfu1t.ffs@fw13/
is the mail where I explained that softirq issue.
The link [1] here is for "the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1]", so I think that's correct because it contains your explanation on how the fixup would work. I will add a "Closes" tag to the other email that you explained the issue. Sounds good?
Regards,
Boqun
PATCH v2 1/1 interrupt: Disable interrupt before modifying hardirq_disable counter
In reply to Boqun FengCurrently a softirq may be pending longer then expected if the triggering interrupt happens in-between hardirq_disable_enter() and _local_interrupt_disable() in local_interrupt_disable():
local_interrupt_disable():
hardirq_disable_enter();
<interrupt>
...
__irq_exit_rcu():
// false because hardirq_disable_count() is not 0
if (.. && !hardirq_disable_count() && ..) {
invoke_softirq();
}
_local_interrupt_disable();, it'll defer the softirq to the next interrupt which can be forever.
The order between hardirq_disable_enter() and _local_interrupt_disable() is to optimize re-disabling interrupts if they are already disabled, but as 1) local_interrupt_disable() is not widely used yet and 2) the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq pending issue.
Because of this fix, the part of saving the current state is separated from irq disabling, and the logic of local_interrupt_disable() becomes:
local_irq_save(flags);
if (counter++ == 0) {
this_cpu(local_interrupt_disable_state) = flags;
}Therefore change the helper function _local_interrupt_disable() to _local_interrupt_save_state() which only saves the current irqflags (when interrupts get disabled the first time).
---
v1 -> v2:
* Use imperative mood in the last paragraph of the change log.
* Add "Closes" tag to the email of the explanation of the issue.
* Apply the RoB tag from Bradley Morgan
include/linux/interrupt_rc.h | 19 ++++++++----------- kernel/softirq.c | 17 ++++------------- 2 files changed, 12 insertions(+), 24 deletions(-)
include/linux/interrupt_rc.h +8 -11
@@ -20,11 +20,8 @@/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); -static __always_inline void __local_interrupt_disable(void) +static __always_inline void __local_interrupt_save_state(unsigned long flags) { -unsigned long flags; - -local_irq_save (flags);raw_cpu_write (local_interrupt_disable_state, flags); }
@@ -36,9 +33,9 @@ static __always_inline void __local_interrupt_enable(void) }#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE -static __always_inline void _local_interrupt_disable(void) +static __always_inline void _local_interrupt_save_state(unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } static __always_inline void _local_interrupt_enable(void)
@@ -46,27 +43,27 @@ static __always_inline void _local_interrupt_enable(void)__local_interrupt_enable (); }#else -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif #else /* !MODULE */ -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif /* !MODULE */ static inline void local_interrupt_disable (void ) {int new_count; +unsigned long flags;WARN_ON_ONCE (in_nmi ()); +local_irq_save (flags); new_count =hardirq_disable_enter (); -/* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ -if ((new_count &HARDIRQ_DISABLE_MASK ) ==HARDIRQ_DISABLE_OFFSET ) -_local_interrupt_disable (); +_local_interrupt_save_state (flags); }static inline void local_interrupt_enable (void )
kernel/softirq.c +4 -13
@@ -91,11 +91,11 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); -void _local_interrupt_disable (void ) +void _local_interrupt_save_state (unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } -EXPORT_SYMBOL (_local_interrupt_disable); +EXPORT_SYMBOL (_local_interrupt_save_state);void _local_interrupt_enable (void ) {
@@ -749,16 +749,7 @@ static inline void __irq_exit_rcu(void)#endif account_hardirq_exit (current);preempt_count_sub (HARDIRQ_OFFSET ); -/* -* Interrupts may happen between hardirq_disable_enter() and -* local_irq_save() in local_interrupt_disable(), if irq_exit() invokes -* softirq here, we may have a softirq handler calling -* local_interrupt_disable() but it won't disable the IRQ because -* hardirq disabling count is already 1, hence we need to prevent -* invoking softirq when a local_interrupt_disable() is ongoing. -*/ -if (!in_interrupt () && !hardirq_disable_count () && -local_softirq_pending ()) { +if (!in_interrupt () &&local_softirq_pending ()) {/* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run softirq.
Signature · 1 line
2.50.1 (Apple Git-155)
The following commit has been merged into the locking/urgent branch of tip:
The following commit has been merged into the locking/urgent branch of tip:
Commit-ID: a155ac8f0c523bd53f412196dcbb104ad1f4595f Gitweb: https://git.kernel.org/tip/a155ac8f0c523bd53f412196dcbb104ad1f4595f Author: Boqun Feng <[email protected]> AuthorDate: Sat, 29 Aug 2026 14:34:12 -07:00 Committer: Thomas Gleixner <[email protected]> CommitterDate: Sun, 30 Aug 2026 08:39:04 +02:00
interrupt: Disable interrupt before modifying hardirq_disable counter
Currently a softirq may be pending longer then expected if the triggering interrupt happens in-between hardirq_disable_enter() and _local_interrupt_disable() in local_interrupt_disable():
local_interrupt_disable():
hardirq_disable_enter();
<interrupt>
...
__irq_exit_rcu():
// false because hardirq_disable_count() is not 0
if (.. && !hardirq_disable_count() && ..) {
invoke_softirq();
}
_local_interrupt_disable();, it'll defer the softirq to the next interrupt which can be forever.
The order between hardirq_disable_enter() and _local_interrupt_disable() is to optimize re-disabling interrupts if they are already disabled, but as 1) local_interrupt_disable() is not widely used yet and 2) the proper way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq pending issue.
Because of this fix, the part of saving the current state is separated from irq disabling, and the logic of local_interrupt_disable() becomes:
local_irq_save(flags);
if (counter++ == 0) {
this_cpu(local_interrupt_disable_state) = flags;
}Therefore change the helper function _local_interrupt_disable() to _local_interrupt_save_state() which only saves the current irqflags (when interrupts get disabled the first time).
--- include/linux/interrupt_rc.h | 19 ++++++++----------- kernel/softirq.c | 17 ++++------------- 2 files changed, 12 insertions(+), 24 deletions(-)
include/linux/interrupt_rc.h +8 -11
@@ -20,11 +20,8 @@/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); -static __always_inline void __local_interrupt_disable(void) +static __always_inline void __local_interrupt_save_state(unsigned long flags) { -unsigned long flags; - -local_irq_save (flags);raw_cpu_write (local_interrupt_disable_state, flags); }
@@ -36,9 +33,9 @@ static __always_inline void __local_interrupt_enable(void) }#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE -static __always_inline void _local_interrupt_disable(void) +static __always_inline void _local_interrupt_save_state(unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } static __always_inline void _local_interrupt_enable(void)
@@ -46,27 +43,27 @@ static __always_inline void _local_interrupt_enable(void)__local_interrupt_enable (); }#else -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif #else /* !MODULE */ -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void _local_interrupt_enable (void );#endif /* !MODULE */ static inline void local_interrupt_disable (void ) {int new_count; +unsigned long flags;WARN_ON_ONCE (in_nmi ()); +local_irq_save (flags); new_count =hardirq_disable_enter (); -/* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ -if ((new_count &HARDIRQ_DISABLE_MASK ) ==HARDIRQ_DISABLE_OFFSET ) -_local_interrupt_disable (); +_local_interrupt_save_state (flags); }static inline void local_interrupt_enable (void )
kernel/softirq.c +4 -13
@@ -91,11 +91,11 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); -void _local_interrupt_disable (void ) +void _local_interrupt_save_state (unsigned long flags) { -__local_interrupt_disable (); +__local_interrupt_save_state (flags); } -EXPORT_SYMBOL (_local_interrupt_disable); +EXPORT_SYMBOL (_local_interrupt_save_state);void _local_interrupt_enable (void ) {
@@ -749,16 +749,7 @@ static inline void __irq_exit_rcu(void)#endif account_hardirq_exit (current);preempt_count_sub (HARDIRQ_OFFSET ); -/* -* Interrupts may happen between hardirq_disable_enter() and -* local_irq_save() in local_interrupt_disable(), if irq_exit() invokes -* softirq here, we may have a softirq handler calling -* local_interrupt_disable() but it won't disable the IRQ because -* hardirq disabling count is already 1, hence we need to prevent -* invoking softirq when a local_interrupt_disable() is ongoing. -*/ -if (!in_interrupt () && !hardirq_disable_count () && -local_softirq_pending ()) { +if (!in_interrupt () &&local_softirq_pending ()) {/* * If we left hrtimers unarmed, make sure to arm them now, * before enabling interrupts to run softirq.
Quoted 172 lines · Boqun Feng
Currently a softirq may be pending longer then expected if the triggering interrupt happens in-between
hardirq_disable_enter()and_local_interrupt_disable()inlocal_interrupt_disable():
local_interrupt_disable():
hardirq_disable_enter();
<interrupt>
...
__irq_exit_rcu():
// false becausehardirq_disable_count()is not 0if (.. && !hardirq_disable_count() && ..) { invoke_softirq(); }
_local_interrupt_disable();, it'll defer the softirq to the next interrupt which can be forever.
The order between
hardirq_disable_enter()and_local_interrupt_disable()
is to optimize re-disabling interrupts if they are already disabled, but
as 1)local_interrupt_disable()is not widely used yet and 2) the proper
way to achieve this optimization may need fixing up the counter at entry/exit time [1], so reverse the order for now to avoid the softirq
pending issue.Because of this fix, the part of saving the current state is separated
from irq disabling, and the logic oflocal_interrupt_disable()becomes:
local_irq_save(flags);
if (counter++ == 0) {this_cpu(local_interrupt_disable_state)= flags; }Therefore change the helper function
_local_interrupt_disable()to_local_interrupt_save_state()which only saves the current irqflags (when interrupts get disabled the first time).Reported-by Thomas Gleixner <[email protected]>Fixes e901c1510e24 ("irq,spin_lock: Add counted interruptdisabling/enabling")
Reviewed-by Bradley Morgan <[email protected]>Signed-off-by Boqun Feng <[email protected]>---
v1 -> v2:* Use imperative mood in the last paragraph of the change log.
* Add "Closes" tag to the email of the explanation of the issue.
* Apply the RoB tag from Bradley Morganinclude/linux/interrupt_rc.h | 19 ++++++++----------- kernel/softirq.c | 17 ++++-------------
2 files changed, 12 insertions(+), 24 deletions(-)include/linux/interrupt_rc.h +0
-0@@ -20,11 +20,8 @@ /* Per-CPU interrupt disabling state forlocal_interrupt_{disable,enable}(). */
DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);include/linux/interrupt_rc.h +1
-1-static __always_inline void __local_interrupt_disable(void) +static __always_inline void __local_interrupt_save_state(unsignedlong flags)
{include/linux/interrupt_rc.h +0
-3-unsigned long flags; - -local_irq_save (flags);
raw_cpu_write(local_interrupt_disable_state, flags); }include/linux/interrupt_rc.h +0
-0@@ -36,9 +33,9 @@ static __always_inline void__local_interrupt_enable (void ) }
#ifndefINSTANTIATE_EXPORTED_INTERRUPT_DISABLEinclude/linux/interrupt_rc.h +1
-1-static __always_inline void _local_interrupt_disable(void) +static __always_inline void _local_interrupt_save_state(unsignedlong flags)
{include/linux/interrupt_rc.h +1
-1-__local_interrupt_disable (); +__local_interrupt_save_state (flags);}
static__always_inlinevoid_local_interrupt_enable(void)include/linux/interrupt_rc.h +1
-1@@ -46,27 +43,27 @@ static __always_inline void_local_interrupt_enable (void ) __local_interrupt_enable(); } #else -extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags); extern void _local_interrupt_enable(void); #endif
#else /* !MODULE */include/linux/interrupt_rc.h +1
-1-extern void _local_interrupt_disable (void ); +extern void _local_interrupt_save_state (unsigned long flags);extern void
_local_interrupt_enable(void);
#endif /* !MODULE */
static inline voidlocal_interrupt_disable(void)
{
intnew_count;include/linux/interrupt_rc.h +1
-0+unsigned long flags;
WARN_ON_ONCE(in_nmi());include/linux/interrupt_rc.h +1
-0+local_irq_save (flags);
new_count=hardirq_disable_enter();include/linux/interrupt_rc.h +0
-1-/* Interrupts can happen here, but it's OK, see
__irq_exit_rcu(). */
-
if ((new_count&HARDIRQ_DISABLE_MASK) ==HARDIRQ_DISABLE_OFFSET)include/linux/interrupt_rc.h +1
-1-_local_interrupt_disable (); +_local_interrupt_save_state (flags);}
static inline voidlocal_interrupt_enable(void)kernel/softirq.c +0
-0@@ -91,11 +91,11 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
DEFINE_PER_CPU(unsigned long,local_interrupt_disable_state);kernel/softirq.c +1
-1-void _local_interrupt_disable (void ) +void _local_interrupt_save_state (unsigned long flags){
kernel/softirq.c +1
-1-__local_interrupt_disable (); +__local_interrupt_save_state (flags);}
kernel/softirq.c +1
-1-EXPORT_SYMBOL (_local_interrupt_disable); +EXPORT_SYMBOL (_local_interrupt_save_state);
void_local_interrupt_enable(void)
{kernel/softirq.c +0
-3@@ -749,16 +749,7 @@ static inline void __irq_exit_rcu(void) #endif account_hardirq_exit(current); preempt_count_sub(HARDIRQ_OFFSET); -/* -* Interrupts may happen between hardirq_disable_enter() and -* local_irq_save() in local_interrupt_disable(), if
irq_exit()invokeskernel/softirq.c +0
-2-* softirq here, we may have a softirq handler calling -* local_interrupt_disable() but it won't disable the IRQ because
kernel/softirq.c +0
-1-* hardirq disabling count is already 1, hence we need to prevent
kernel/softirq.c +0
-1-* invoking softirq when a local_interrupt_disable() is ongoing.
kernel/softirq.c +1
-3-*/ -if (!in_interrupt () && !hardirq_disable_count () && - local_softirq_pending()) { +if (!in_interrupt () &&local_softirq_pending ()) {/*
* If we left hrtimers unarmed, make sure to arm them now,
* before enabling interrupts to run softirq.