Refcounted interrupt disable and SpinLockIrq for Rust
Boqun Feng
maintainerv5 cover Refcounted interrupt disable and SpinLockIrq for Rust
Hi Peter & Ingo,
Please take a look at this updated series for SpinLockIrq.
Changes since v4:
* Addressed SoB issues spotted by Peter.
* Added underflow/overflow checking for interrupt disabling counter as suggested by Shrikanth.
* Added a max nesting level per suggestion from Shrikanth.
* Document/comment updates.
v4: https://lore.kernel.org/lkml/[email protected]/
Regards,
Boqun
Boqun Feng (10):
preempt: Introduce HARDIRQ_DISABLE_BITS
preempt: Introduce __preempt_count_{sub,add}_return()
irq & spin_lock: Add counted interrupt disabling/enabling
irq: Add max local_interrupt_disable() nesting level kunit test case
locking: Switch to _irq_{disable,enable}() variants in cleanup guards
sched: Remove the unused preempt_offset parameter of __cant_sleep()
sched: Avoid signed comparison of preempt_count() in __cant_migrate()
preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers
Heiko Carstens (1):
s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
Joel Fernandes (1):
preempt: Track NMI nesting to separate per-CPU counter
Lyude Paul (6):
openrisc: Include <linux/cpumask.h> in smp.h
irq: Add KUnit test for refcounted interrupt enable/disable
rust: Introduce interrupt module
rust: sync: Use super::* in spinlock.rs
rust: sync: Add SpinLockIrq
rust: sync: Introduce SpinLockIrq::lock_with() and friends
arch/arm64/Kconfig | 1 + arch/arm64/include/asm/preempt.h | 20 ++ arch/openrisc/include/asm/smp.h | 2 + arch/s390/Kconfig | 1 + arch/s390/include/asm/lowcore.h | 13 +- arch/s390/include/asm/preempt.h | 51 +-- arch/x86/Kconfig | 1 + arch/x86/include/asm/preempt.h | 61 +++- arch/x86/kernel/cpu/common.c | 2 +- include/asm-generic/preempt.h | 14 + include/linux/hardirq.h | 40 ++- include/linux/interrupt_rc.h | 95 +++++ include/linux/kernel.h | 4 +- include/linux/preempt.h | 44 ++- include/linux/spinlock.h | 49 ++- include/linux/spinlock_api_smp.h | 41 +++ include/linux/spinlock_api_up.h | 15 + include/linux/spinlock_rt.h | 18 + kernel/Kconfig.preempt | 4 + kernel/irq/Makefile | 1 + kernel/irq/refcount_interrupt_test.c | 131 +++++++ kernel/locking/spinlock.c | 31 ++ kernel/sched/core.c | 18 +- kernel/sched/fair.c | 12 +- kernel/softirq.c | 36 +- rust/helpers/helpers.c | 1 + rust/helpers/interrupt.c | 18 + rust/helpers/spinlock.c | 15 + rust/helpers/sync.c | 5 + rust/kernel/interrupt.rs | 89 +++++ rust/kernel/lib.rs | 1 + rust/kernel/sync.rs | 9 +- rust/kernel/sync/lock/global.rs | 3 + rust/kernel/sync/lock/spinlock.rs | 329 +++++++++++++++++- .../testing/selftests/bpf/bpf_experimental.h | 7 +- 35 files changed, 1092 insertions(+), 90 deletions(-) create mode 100644 include/linux/interrupt_rc.h create mode 100644 kernel/irq/refcount_interrupt_test.c create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/kernel/interrupt.rs
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 1/18 preempt: Track NMI nesting to separate per-CPU counter
From: Joel Fernandes <[email protected]>
Move NMI nesting tracking from the preempt_count bits to a separate per-CPU counter (nmi_nesting). This is to free up the NMI bits in the preempt_count, allowing those bits to be repurposed for other uses.
Reduce NMI_BITS from 4 to 1, using it only to detect if we're in an NMI. The per-CPU counter currently caps nesting at 15.
[boqun: Address Steven Rostedt's comment on the BUG_ON() condition] [boqun: Use preempt_count_set() in __nmi_exit() to avoid underflow]
--- include/linux/hardirq.h | 17 +++++++++++++---- include/linux/preempt.h | 9 +++++++-- kernel/softirq.c | 2 ++ tools/testing/selftests/bpf/bpf_experimental.h | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-)
include/linux/hardirq.h +13 -4
@@ -10,6 +10,8 @@#include <linux/vtime.h> #include <asm/hardirq.h> +DECLARE_PER_CPU(unsigned int, nmi_nesting); +extern void synchronize_irq (unsigned int irq);extern bool synchronize_hardirq (unsigned int irq);
@@ -102,14 +104,17 @@ void irq_exit_rcu(void);*/ /* -* nmi_enter() can nest up to 15 times; see NMI_BITS. +* nmi_enter() can nest - nesting is tracked in a per-CPU counter. */ #define __nmi_enter () \ do { \ lockdep_off(); \ arch_nmi_enter(); \ - BUG_ON(in_nmi() == NMI_MASK); \ - __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ +/* Maximum NMI nesting is 15. */ \ +BUG_ON (__this_cpu_read (nmi_nesting) >=15 ); \ +__this_cpu_inc (nmi_nesting); \ +__preempt_count_add (HARDIRQ_OFFSET ); \ +preempt_count_set (preempt_count () |NMI_MASK ); \ } while (0)#define nmi_enter () \
@@ -124,8 +129,12 @@ void irq_exit_rcu(void);#define __nmi_exit () \ do { \ + unsigned int nesting; \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + nesting = __this_cpu_dec_return(nmi_nesting); \ + if (!nesting) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0)
include/linux/preempt.h +7 -2
@@ -17,6 +17,8 @@* * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) +* - bits 16-19 are the hardirq count (max # of hardirqs: 16) +* - bit 20 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with
@@ -24,16 +26,19 @@* there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * +* NMI nesting depth is tracked in a separate per-CPU variable +* (nmi_nesting) to save bits in preempt_count. +* * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00f00000 +* NMI_MASK: 0x00100000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8#define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
kernel/softirq.c +2 -0
@@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);EXPORT_PER_CPU_SYMBOL_GPL (hardirq_context);#endif +DEFINE_PER_CPU(unsigned int, nmi_nesting); +/* * SOFTIRQ_OFFSET usage: *
tools/testing/selftests/bpf/bpf_experimental.h +1 -1
@@ -367,7 +367,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,#define PREEMPT_BITS 8#define SOFTIRQ_BITS 8#define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS 1#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 2/18 preempt: Introduce HARDIRQ_DISABLE_BITS
In reply to Boqun FengIn order to support preempt_disable()-like interrupt disabling, that is, using part of preempt_count() to track interrupt disabling nesting level, change the preempt_count() layout to contain 8-bit HARDIRQ_DISABLE count.
--- include/linux/preempt.h | 16 +++++++++++----- tools/testing/selftests/bpf/bpf_experimental.h | 5 ++++- 2 files changed, 15 insertions(+), 6 deletions(-)
include/linux/preempt.h +11 -5
@@ -17,8 +17,9 @@* * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) -* - bits 16-19 are the hardirq count (max # of hardirqs: 16) -* - bit 20 is the NMI flag (no nesting count, tracked separately) +* - bits 16-23 are the hardirq disable count (max # of hardirq disable: 256) +* - bits 24-27 are the hardirq count (max # of hardirqs: 16) +* - bit 28 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with
@@ -31,29 +32,34 @@* * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 -* HARDIRQ_MASK: 0x000f0000 -* NMI_MASK: 0x00100000 +* HARDIRQ_DISABLE_MASK: 0x00ff0000 +* HARDIRQ_MASK: 0x0f000000 +* NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8#define HARDIRQ_BITS 4#define NMI_BITS 1#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS)#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)#define __IRQ_MASK (x) ((1UL << (x))-1)#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT)#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT)#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)#define NMI_OFFSET (1UL << NMI_SHIFT)
tools/testing/selftests/bpf/bpf_experimental.h +4 -1
@@ -366,17 +366,20 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,#define PREEMPT_BITS 8#define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8#define HARDIRQ_BITS 4#define NMI_BITS 1#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS)#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)#define __IRQ_MASK (x) ((1UL << (x))-1)#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT)#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 3/18 preempt: Introduce __preempt_count_{sub,add}_return()
In reply to Boqun FengIn order to use preempt_count() to track the interrupt disable nesting level, __preempt_count_{add,sub}_return() are introduced, as their names suggest, these primitives return the new value of the preempt_count() after changing it. The following example shows the usage of it in local_interrupt_disable():
// increase the HARDIRQ_DISABLE bit
new_count = __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET);
// if it's the first-time increment, then disable the interrupt
// at hardware level.
if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) {
local_irq_save(flags);
raw_cpu_write(local_interrupt_disable_state, flags);
}Having these primitives will avoid a read of preempt_count() after changing preempt_count() on certain architectures.
--- arch/arm64/include/asm/preempt.h | 20 ++++++++++++++++++++ arch/s390/include/asm/preempt.h | 10 ++++++++++ arch/x86/include/asm/preempt.h | 10 ++++++++++ include/asm-generic/preempt.h | 14 ++++++++++++++ 4 files changed, 54 insertions(+)
arch/arm64/include/asm/preempt.h +20 -0
@@ -55,6 +55,26 @@ static inline void __preempt_count_sub(int val)WRITE_ONCE (current_thread_info ()->preempt.count, pc); } +static inline int __preempt_count_add_return (int val) +{ +u32 pc =READ_ONCE (current_thread_info ()->preempt.count); + + pc += val; +WRITE_ONCE (current_thread_info ()->preempt.count, pc); + +return pc; +} + +static inline int __preempt_count_sub_return (int val) +{ +u32 pc =READ_ONCE (current_thread_info ()->preempt.count); + + pc -= val; +WRITE_ONCE (current_thread_info ()->preempt.count, pc); + +return pc; +} +static inline bool __preempt_count_dec_and_test (void ) {struct thread_info *ti =current_thread_info ();
arch/s390/include/asm/preempt.h +10 -0
@@ -139,6 +139,16 @@ static __always_inline bool should_resched(int preempt_offset)return unlikely (READ_ONCE (get_lowcore ()->preempt_count) == preempt_offset); } +static __always_inline int __preempt_count_add_return(int val) +{ +return val +__atomic_add (val, &get_lowcore ()->preempt_count); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ +return __preempt_count_add_return (-val); +} +#define init_task_preempt_count (p) do { } while (0)/* Deferred to CPU bringup time */ #define init_idle_preempt_count (p, cpu) do { } while (0)
arch/x86/include/asm/preempt.h +10 -0
@@ -85,6 +85,16 @@ static __always_inline void __preempt_count_sub(int val)raw_cpu_add_4 (__preempt_count, -val); } +static __always_inline int __preempt_count_add_return(int val) +{ +return raw_cpu_add_return_4 (__preempt_count, val); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ +return raw_cpu_add_return_4 (__preempt_count, -val); +} +/* * Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule * a decrement which hits zero means we have no preempt_count and should
include/asm-generic/preempt.h +14 -0
@@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val) *preempt_count_ptr () -= val; } +static __always_inline int __preempt_count_add_return(int val) +{ + *preempt_count_ptr () += val; + +return *preempt_count_ptr (); +} + +static __always_inline int __preempt_count_sub_return(int val) +{ + *preempt_count_ptr () -= val; + +return *preempt_count_ptr (); +} + static __always_inline bool __preempt_count_dec_and_test(void) {/*
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 4/18 openrisc: Include <linux/cpumask.h> in smp.h
In reply to Boqun FengFrom: Lyude Paul <[email protected]>
While OpenRISC currently doesn't fail to build upstream, it appears that including <asm/smp.h> in the right headers is enough to break that - primarily because OpenRISC's asm/smp.h header doesn't actually provide any definition for struct cpumask. Which means the only reason we aren't failing to build the kernel is because we've been lucky enough that every spot including asm/smp.h already has definitions for struct cpumask pulled in.
This became evident when trying to work on a patch series for adding ref-counted interrupt enable/disable to the kernel, where introducing a new interrupt_rc.h header suddenly introduced a build error on OpenRISC:
In file included from include/linux/interrupt_rc.h:17,
from include/linux/spinlock.h:60,
from include/linux/mmzone.h:8,
from include/linux/gfp.h:7,
from include/linux/mm.h:7,
from arch/openrisc/include/asm/pgalloc.h:20,
from arch/openrisc/include/asm/io.h:18,
from include/linux/io.h:12,
from drivers/irqchip/irq-ompic.c:61:
arch/openrisc/include/asm/smp.h:21:59: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration
21 | extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
| ^~~~~~~
arch/openrisc/include/asm/smp.h:23:54: warning: 'struct cpumask' declared inside parameter list will not be visible outside of this definition or declaration
23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
| ^~~~~~~
drivers/irqchip/irq-ompic.c: In function 'ompic_of_init':
Quoted 1 line
Quoted 1 line
drivers/irqchip/irq-ompic.c:191:28: error: passing argument 1 of
'set_smp_cross_call' from incompatible pointer type [-Werror=incompatible-pointer-types]
191 | set_smp_cross_call(ompic_raise_softirq);
| ^~~~~~~~~~~~~~~~~~~
| |
| void (*)(const struct cpumask *, unsigned int)
arch/openrisc/include/asm/smp.h:23:32: note: expected 'void (*)(const struct cpumask *, unsigned int)' but argument is of type 'void
(*)(const struct cpumask *, unsigned int)'
23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
To fix this, let's take an example from the smp.h headers of other architectures (x86, hexagon, arm64, probably more): just include linux/cpumask.h at the top.
--- arch/openrisc/include/asm/smp.h | 2 ++ 1 file changed, 2 insertions(+)
arch/openrisc/include/asm/smp.h +2 -0
@@ -9,6 +9,8 @@#ifndef __ASM_OPENRISC_SMP_H#define __ASM_OPENRISC_SMP_H +#include <linux/cpumask.h> +#include <asm/spr.h> #include <asm/spr_defs.h>
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 5/18 irq & spin_lock: Add counted interrupt disabling/enabling
In reply to Boqun FengCurrently the nested interrupt disabling and enabling is represented by _irqsave() and _irqrestore() APIs, which are relatively unsafe, for example:
<interrupts are enabled as beginning> spin_lock_irqsave(l1, flag1); spin_lock_irqsave(l2, flag2); spin_unlock_irqrestore(l1, flags1); <l2 is still held but interrupts are enabled> // accesses to interrupt-disable protected data will cause races
This is even easier to trigger with guard facilities:
unsigned long flag2;
scoped_guard(spin_lock_irqsave, l1) {
spin_lock_irqsave(l2, flag2);
}
// l2 locked but interrupts are enabled.
spin_unlock_irqrestore(l2, flag2);(Hand-to-hand locking critical sections are not uncommon for a fine-grained lock design)
And because of this unsafety, Rust cannot easily wrap the interrupt-disabling locks in a safe API, which complicates the design.
To resolve this, introduce a new set of interrupt disabling APIs:
* local_interrupt_disable();
* local_interrupt_enable();
They work like local_irq_save() and local_irq_restore() except that 1) the outermost local_interrupt_disable() call saves the interrupt state into a per-CPU variable, so that the outermost local_interrupt_enable() can restore the state, and 2) a per-CPU counter is added to record the nest level of these calls, so that interrupts are not accidentally enabled inside the outermost critical section.
Also add the corresponding spin_lock primitives: spin_lock_irq_disable() and spin_unlock_irq_enable(), as a result, code as follows:
spin_lock_irq_disable(l1); spin_lock_irq_disable(l2); spin_unlock_irq_enable(l1); // Interrupts are still disabled. spin_unlock_irq_enable(l2);
doesn't have the issue that interrupts are accidentally enabled.
This also makes the wrapper of interrupt-disabling locks on Rust easier to design.
--- include/linux/interrupt_rc.h | 95 ++++++++++++++++++++++++++++++++ include/linux/preempt.h | 4 ++ include/linux/spinlock.h | 23 ++++++++ include/linux/spinlock_api_smp.h | 41 ++++++++++++++ include/linux/spinlock_api_up.h | 15 +++++ include/linux/spinlock_rt.h | 18 ++++++ kernel/locking/spinlock.c | 31 +++++++++++ kernel/softirq.c | 28 +++++++++- 8 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 include/linux/interrupt_rc.h
include/linux/interrupt_rc.h new file +95 -0
@@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_INTERRUPT_RC_H +#define __LINUX_INTERRUPT_RC_H + +/* +* include/linux/interrupt_rc.h - refcounted local processor interrupt +* management. +* +* Since the implementation of this API currently depends on +* local_irq_save()/local_irq_restore(), we split this into its own header to +* make it easier to include without hitting circular header dependencies. +*/ + +#include <linux/irqflags.h> +#include <linux/debug_locks.h> +#include <linux/preempt.h> +#include <linux/processor.h> +#include <linux/smp.h> + +#ifndef MODULE +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); + +static __always_inline void __local_interrupt_disable(void) +{ +unsigned long flags; + +local_irq_save (flags); +raw_cpu_write (local_interrupt_disable_state, flags); +} + +static __always_inline void __local_interrupt_enable(void) +{ +unsigned long flags =raw_cpu_read (local_interrupt_disable_state); + +local_irq_restore (flags); +} + +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE +static __always_inline void _local_interrupt_disable(void) +{ +__local_interrupt_disable (); +} + +static __always_inline void _local_interrupt_enable(void) +{ +__local_interrupt_enable (); +} +#else +extern void _local_interrupt_disable (void ); +extern void _local_interrupt_enable (void ); +#endif + +#else /* !MODULE */ +extern void _local_interrupt_disable (void ); +extern void _local_interrupt_enable (void ); +#endif /* !MODULE */ + +static inline void local_interrupt_disable (void ) +{ +int new_count; + +WARN_ON_ONCE (in_nmi ()); + + new_count =hardirq_disable_enter (); + +/* Is hardirq disable count overflow soon? */ +if (IS_ENABLED (CONFIG_DEBUG_PREEMPT ) && +DEBUG_LOCKS_WARN_ON ((new_count &HARDIRQ_DISABLE_MASK ) + + (10 <<HARDIRQ_DISABLE_SHIFT ) >= +HARDIRQ_DISABLE_MASK )) +return ; + +/* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ + +if ((new_count &HARDIRQ_DISABLE_MASK ) ==HARDIRQ_DISABLE_OFFSET ) +_local_interrupt_disable (); +} + +static inline void local_interrupt_enable (void ) +{ +int new_count; + +/* Unpaired local_interrupt_enable()? Warn and abort. */ +if (IS_ENABLED (CONFIG_DEBUG_PREEMPT ) && +DEBUG_LOCKS_WARN_ON ((preempt_count () &HARDIRQ_DISABLE_MASK ) ==0 )) +return ; + + new_count =hardirq_disable_exit (); + +if ((new_count &HARDIRQ_DISABLE_MASK ) ==0 ) +_local_interrupt_enable (); +} + +#endif /* !__LINUX_INTERRUPT_RC_H */
include/linux/preempt.h +4 -0
@@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context_level(void)#define in_softirq () (softirq_count())#define in_interrupt () (irq_count()) +#define hardirq_disable_count () ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT) +#define hardirq_disable_enter () __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) +#define hardirq_disable_exit () __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) +/* * The preempt_count offset after preempt_disable(); */
include/linux/spinlock.h +23 -0
@@ -57,6 +57,7 @@#include <linux/linkage.h> #include <linux/compiler.h> #include <linux/irqflags.h> +#include <linux/interrupt_rc.h> #include <linux/thread_info.h> #include <linux/stringify.h> #include <linux/bottom_half.h>
@@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)#endif #define raw_spin_lock_irq (lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq_disable (lock) _raw_spin_lock_irq_disable(lock)#define raw_spin_lock_bh (lock) _raw_spin_lock_bh(lock)#define raw_spin_unlock (lock) _raw_spin_unlock(lock)#define raw_spin_unlock_irq (lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq_enable (lock) _raw_spin_unlock_irq_enable(lock)#define raw_spin_unlock_irqrestore (lock, flags) \ do { \
@@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)#define raw_spin_trylock_irqsave (lock, flags) _raw_spin_trylock_irqsave(lock, &(flags)) +#define raw_spin_trylock_irq_disable (lock) _raw_spin_trylock_irq_disable(lock) +#ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ #include <linux/rwlock.h>
@@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)raw_spin_lock_irq (&lock->rlock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) +__acquires (lock) __no_context_analysis +{ +raw_spin_lock_irq_disable (&lock->rlock); +} +#define spin_lock_irqsave (lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \
@@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)raw_spin_unlock_irq (&lock->rlock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) +__releases (lock) __no_context_analysis +{ +raw_spin_unlock_irq_enable (&lock->rlock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)__releases (lock) __no_context_analysis {
@@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon }#define spin_trylock_irqsave (lock, flags) _spin_trylock_irqsave(lock, &(flags)) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) +__cond_acquires (true, lock) __no_context_analysis +{ +return raw_spin_trylock_irq_disable (&lock->rlock); +} +/** * spin_is_locked() - Check whether a spinlock is locked. * @lock: Pointer to the spinlock.
include/linux/spinlock_api_smp.h +41 -0
83 lines not on this page · raw message
include/linux/spinlock_api_up.h +15 -0
@@ -42,6 +42,9 @@#define __LOCK_IRQSAVE (lock, flags, ...) \ do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) +#define __LOCK_IRQ_DISABLE (lock, ...) \ + do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) +#define ___UNLOCK_ (lock) \ do { __release(lock); (void)(lock); } while (0)
@@ -61,6 +64,9 @@#define __UNLOCK_IRQRESTORE (lock, flags, ...) \ do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) +#define __UNLOCK_IRQ_ENABLE (lock, ...) \ + do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0) +#define _raw_spin_lock (lock) __LOCK(lock)#define _raw_spin_lock_nested (lock, subclass) __LOCK(lock)#define _raw_read_lock (lock) __LOCK(lock, shared)
@@ -70,6 +76,7 @@#define _raw_read_lock_bh (lock) __LOCK_BH(lock, shared)#define _raw_write_lock_bh (lock) __LOCK_BH(lock)#define _raw_spin_lock_irq (lock) __LOCK_IRQ(lock) +#define _raw_spin_lock_irq_disable (lock) __LOCK_IRQ_DISABLE(lock)#define _raw_read_lock_irq (lock) __LOCK_IRQ(lock, shared)#define _raw_write_lock_irq (lock) __LOCK_IRQ(lock)#define _raw_spin_lock_irqsave (lock, flags) __LOCK_IRQSAVE(lock, flags)
@@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock)return 1 ; } +static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock) +__cond_acquires (true, lock) +{ +__LOCK_IRQ_DISABLE (lock); +return 1 ; +} + static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)__cond_acquires (true, lock) {
@@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l#define _raw_write_unlock_bh (lock) __UNLOCK_BH(lock)#define _raw_read_unlock_bh (lock) __UNLOCK_BH(lock, shared)#define _raw_spin_unlock_irq (lock) __UNLOCK_IRQ(lock) +#define _raw_spin_unlock_irq_enable (lock) __UNLOCK_IRQ_ENABLE(lock)#define _raw_read_unlock_irq (lock) __UNLOCK_IRQ(lock, shared)#define _raw_write_unlock_irq (lock) __UNLOCK_IRQ(lock)#define _raw_spin_unlock_irqrestore (lock, flags) \
include/linux/spinlock_rt.h +18 -0
@@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)rt_spin_lock (lock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) +__acquires (lock) +{ +rt_spin_lock (lock); +} +#define spin_lock_irqsave (lock, flags) \ do { \ typecheck(unsigned long, flags); \
@@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)rt_spin_unlock (lock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) +__releases (lock) +{ +rt_spin_unlock (lock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock)
@@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,#define spin_trylock (lock) rt_spin_trylock(lock) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) +__cond_acquires (true, lock) +{ +return rt_spin_trylock (lock); +} +#define spin_trylock_bh (lock) rt_spin_trylock_bh(lock)#define spin_trylock_irq (lock) rt_spin_trylock(lock)
kernel/locking/spinlock.c +31 -0
@@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \*/ BUILD_LOCK_OPS (spin, raw_spinlock, __acquires); +/* No rwlock_t variants for now, so just build this function by hand */ +static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ +for (;;) { +preempt_disable (); +local_interrupt_disable (); +if (likely (do_raw_spin_trylock (lock))) +break ; +local_interrupt_enable (); +preempt_enable (); + +arch_spin_relax (&lock->raw_lock); + } +} +#ifndef CONFIG_PREEMPT_RT BUILD_LOCK_OPS (read, rwlock, __acquires_shared);BUILD_LOCK_OPS (write, rwlock, __acquires);
@@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)EXPORT_SYMBOL (_raw_spin_lock_irq);#endif +#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ +noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ +__raw_spin_lock_irq_disable (lock); +} +EXPORT_SYMBOL_GPL (_raw_spin_lock_irq_disable); +#endif +#ifndef CONFIG_INLINE_SPIN_LOCK_BH noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) {
@@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)EXPORT_SYMBOL (_raw_spin_unlock_irq);#endif +#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ +noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) +{ +__raw_spin_unlock_irq_enable (lock); +} +EXPORT_SYMBOL_GPL (_raw_spin_unlock_irq_enable); +#endif +#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) {
kernel/softirq.c +26 -2
@@ -9,6 +9,7 @@#define pr_fmt (fmt) KBUILD_MODNAME ": " fmt +#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE #include <linux/export.h> #include <linux/kernel_stat.h> #include <linux/interrupt.h>
@@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);EXPORT_PER_CPU_SYMBOL_GPL (hardirq_context);#endif +DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); + +void _local_interrupt_disable (void ) +{ +__local_interrupt_disable (); +} +EXPORT_SYMBOL (_local_interrupt_disable); + +void _local_interrupt_enable (void ) +{ +__local_interrupt_enable (); +} +EXPORT_SYMBOL (_local_interrupt_enable); + DEFINE_PER_CPU(unsigned int, nmi_nesting);/*
@@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void)#endif account_hardirq_exit (current);preempt_count_sub (HARDIRQ_OFFSET ); -if (!in_interrupt () &&local_softirq_pending ()) { +/* +* Interrupts may happen between hardirq_disable_enter() and +* local_irq_save() in local_interrupt_disable(), if irq_exit() invokes +* softirq here, we may have a softirq handler calling +* local_interrupt_disable() but it won't disable the IRQ because +* hardirq disabling count is already 1, hence we need to prevent +* invoking softirq when a local_interrupt_disable() is ongoing. +*/ +if (!in_interrupt () && !hardirq_disable_count () && +local_softirq_pending ()) {/* * If we left hrtimers unarmed, make sure to arm them now, -* before enabling interrupts to run SoftIRQ. +* before enabling interrupts to run softirq. */ hrtimer_rearm_deferred ();invoke_softirq ();
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 6/18 irq: Add KUnit test for refcounted interrupt enable/disable
In reply to Boqun FengFrom: Lyude Paul <[email protected]>
While making changes to the refcounted interrupt patch series, at some point on my local branch I broke something and ended up writing some kunit tests for testing refcounted interrupts as a result. So, let's include these tests now that we have refcounted interrupts.
--- kernel/irq/Makefile | 1 + kernel/irq/refcount_interrupt_test.c | 109 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 kernel/irq/refcount_interrupt_test.c
kernel/irq/Makefile +1 -0
@@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o +obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o
kernel/irq/refcount_interrupt_test.c new file +109 -0
109 lines not on this page · raw message
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 7/18 irq: Add max local_interrupt_disable() nesting level kunit test case
In reply to Boqun FengTo confirm the max nesting level of local_interrupt_disable() works, a kunit test is added to the whole test suite.
Note that when DEBUG_PREEMPT=y, it'll generate a warning which is expected.
--- kernel/irq/refcount_interrupt_test.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
kernel/irq/refcount_interrupt_test.c +22 -0
@@ -52,6 +52,27 @@ static void test_multiple_irq_change(struct kunit *test)TEST_IRQ_ON (); } +static void test_max_nesting_irq_change (struct kunit *test) +{ +for (int i =0 ; i <__IRQ_MASK (HARDIRQ_DISABLE_BITS ); i++) { +local_interrupt_disable (); +TEST_IRQ_OFF (); + } + + +for (int i =0 ; i <__IRQ_MASK (HARDIRQ_DISABLE_BITS ); i++) { +TEST_IRQ_OFF (); +local_interrupt_enable (); + } + +TEST_IRQ_ON (); + +local_interrupt_disable (); +TEST_IRQ_OFF (); +local_interrupt_enable (); +TEST_IRQ_ON (); +} +static void test_irq_save (struct kunit *test) {unsigned long flags;
@@ -79,6 +100,7 @@ static struct kunit_case test_cases[] = {KUNIT_CASE (test_single_irq_change),KUNIT_CASE (test_nested_irq_change),KUNIT_CASE (test_multiple_irq_change), +KUNIT_CASE (test_max_nesting_irq_change), KUNIT_CASE(test_irq_save), {}, };
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 8/18 locking: Switch to _irq_{disable,enable}() variants in cleanup guards
In reply to Boqun FengThe semantics of various IRQ disabling guards match what *_irq_{disable,enable}() provide, i.e. the interrupt disabling is properly nested, therefore it's OK to switch to use *_irq_{disable,enable}() primitives.
[boqun: Adjust the user-side changes in do_sched_cfs_*_timer() provided by Peter and Lyude]
--- include/linux/spinlock.h | 26 ++++++++++++-------------- kernel/sched/fair.c | 12 ++++++------ 2 files changed, 18 insertions(+), 20 deletions(-)
include/linux/spinlock.h +12 -14
@@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acquires(_T), __releases(*(raw#define class_raw_spinlock_nested_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, _T)DEFINE_LOCK_GUARD_1 (raw_spinlock_irq, raw_spinlock_t, -raw_spin_lock_irq (_T->lock), - raw_spin_unlock_irq(_T->lock)) +raw_spin_lock_irq_disable (_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(raw_spinlock_t **)_T))#define class_raw_spinlock_irq_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, _T) -DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))#define class_raw_spinlock_irq_try_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, _T)
@@ -592,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acquires(_T), __releases(*(raw#define class_raw_spinlock_bh_try_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, _T)DEFINE_LOCK_GUARD_1 (raw_spinlock_irqsave, raw_spinlock_t, -raw_spin_lock_irqsave (_T->lock, _T->flags), -raw_spin_unlock_irqrestore (_T->lock, _T->flags), - unsigned long flags) +raw_spin_lock_irq_disable (_T->lock), + raw_spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __releases(*(raw_spinlock_t **)_T))#define class_raw_spinlock_irqsave_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, _T)DEFINE_LOCK_GUARD_1_COND (raw_spinlock_irqsave, _try, - raw_spin_trylock_irqsave(_T->lock, _T->flags)) + raw_spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))#define class_raw_spinlock_irqsave_try_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T)
@@ -618,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_T), __releases(*(spinlock_t#define class_spinlock_try_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_try, _T)DEFINE_LOCK_GUARD_1 (spinlock_irq, spinlock_t, -spin_lock_irq (_T->lock), - spin_unlock_irq(_T->lock)) +spin_lock_irq_disable (_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spinlock_t **)_T))#define class_spinlock_irq_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq, _T)DEFINE_LOCK_GUARD_1_COND (spinlock_irq, _try, - spin_trylock_irq(_T->lock)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(spinlock_t **)_T))#define class_spinlock_irq_try_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq_try, _T)
@@ -640,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquires(_T), __releases(*(spinloc#define class_spinlock_bh_try_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh_try, _T)DEFINE_LOCK_GUARD_1 (spinlock_irqsave, spinlock_t, -spin_lock_irqsave (_T->lock, _T->flags), -spin_unlock_irqrestore (_T->lock, _T->flags), - unsigned long flags) +spin_lock_irq_disable (_T->lock), + spin_unlock_irq_enable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(spinlock_t **)_T))#define class_spinlock_irqsave_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave, _T)DEFINE_LOCK_GUARD_1_COND (spinlock_irqsave, _try, - spin_trylock_irqsave(_T->lock, _T->flags)) + spin_trylock_irq_disable(_T->lock)) DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T))#define class_spinlock_irqsave_try_constructor (_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T)
kernel/sched/fair.c +6 -6
@@ -7120,7 +7120,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)* period the timer is deactivated until scheduling resumes; cfs_b->idle is * used to track this state. */ -static int do_sched_cfs_period_timer (struct cfs_bandwidth *cfs_b,int overrun,unsigned long flags) +static int do_sched_cfs_period_timer (struct cfs_bandwidth *cfs_b,int overrun) __must_hold(&cfs_b->lock) {int throttled;
@@ -7155,10 +7155,10 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u* This check is repeated as we release cfs_b->lock while we unthrottle. */ while (throttled && cfs_b->runtime >0 ) { -raw_spin_unlock_irqrestore (&cfs_b->lock, flags); +raw_spin_unlock_irq_enable (&cfs_b->lock);/* we can't nest cfs_b->lock while distributing bandwidth */ throttled =distribute_cfs_runtime (cfs_b); -raw_spin_lock_irqsave (&cfs_b->lock, flags); +raw_spin_lock_irq_disable (&cfs_b->lock); }/*
@@ -7266,7 +7266,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)static void do_sched_cfs_slack_timer (struct cfs_bandwidth *cfs_b) {/* confirm we're still not at a refresh boundary */ - scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { + scoped_guard(raw_spinlock_irq, &cfs_b->lock) {u64 runtime =0 , slice =sched_cfs_bandwidth_slice (); cfs_b->slack_started = false;
@@ -7351,14 +7351,14 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)int idle =0 ;int count =0 ; -CLASS (raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); +guard (raw_spinlock_irq)(&cfs_b->lock);for (;;) { overrun =hrtimer_forward_now (timer, cfs_b->period);if (!overrun)break ; - idle =do_sched_cfs_period_timer (cfs_b, overrun, cfsb_guard.flags); + idle =do_sched_cfs_period_timer (cfs_b, overrun);if (++count >3 ) {u64 new, old =ktime_to_ns (cfs_b->period);
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 9/18 sched: Remove the unused preempt_offset parameter of __cant_sleep()
In reply to Boqun FengThe preempt_offset is always 0 in all the callsites of __cant_sleep(), hence remove it. It also allows us to clear up the code a bit by no longer using a "preempt_count() > .." comparison.
--- include/linux/kernel.h | 4 ++-- kernel/sched/core.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
include/linux/kernel.h +2 -2
@@ -72,7 +72,7 @@ extern int dynamic_might_resched(void);#ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void __might_resched (const char *file,int line,unsigned int offsets);extern void __might_sleep (const char *file,int line); -extern void __cant_sleep (const char *file,int line,int preempt_offset); +extern void __cant_sleep (const char *file,int line);extern void __cant_migrate (const char *file,int line);/**
@@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line);* this macro will print a stack trace if it is executed with preemption enabled */ # define cant_sleep () \ - do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) + do { __cant_sleep(__FILE__, __LINE__); } while (0)# define sched_annotate_sleep () (current->task_state_change = 0)/**
kernel/sched/core.c +2 -2
@@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsigned int offsets) }EXPORT_SYMBOL (__might_resched); -void __cant_sleep (const char *file,int line,int preempt_offset) +void __cant_sleep (const char *file,int line) {static unsigned long prev_jiffy;
@@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int preempt_offset)if (!IS_ENABLED (CONFIG_PREEMPT_COUNT ))return ; -if (preempt_count () > preempt_offset) +if (preempt_count ())return ;if (time_before (jiffies, prev_jiffy +HZ ) && prev_jiffy)
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 10/18 sched: Avoid signed comparison of preempt_count() in __cant_migrate()
In reply to Boqun FengCurrently preempt_count() is always a non-negative int on all archs (PREEMPT_NEED_RESCHED archs will mask out the MSB when returning preempt_count()), hence the checking in __cant_migrate() is in fact just checking whether preempt_count() is 0 or not. In a future change, we are going to use all the 32 bits of preempt_count(), which would make negative int values possible from preempt_count(). Therefore convert the "> 0" comparison into a zero check to prepare for the future change. No functional changes are intended.
--- kernel/sched/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/sched/core.c +1 -1
@@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line)if (!IS_ENABLED (CONFIG_PREEMPT_COUNT ))return ; -if (preempt_count () >0 ) +if (preempt_count ())return ;if (time_before (jiffies, prev_jiffy +HZ ) && prev_jiffy)
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 11/18 preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
In reply to Boqun FengWith the changes that enable preempt count to track IRQ disabling nesting, we don't have enough bits in 32-bit preempt count implementation, as a result we move NMI nesting bits out of the 32-bit preempt count. However on the architectures that can support 64-bit preempt count implementation, we can keep the NMI nesting bits in the 32-bit preempt count and avoid maintaining NMI nesting bits outside of the same cache line.
Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow architectures to select this. Note that under this Kconfig, preempt count is maintained in a 64-bit word however preempt_count() still remains as an int because all the effective bits still fit in (previously we mask out NEED_RESCHED bit in preempt_count()). This should make no functional changes for existing preempt_count() users.
Enable this for x86_64 along with the introduction of the Kconfig.
[boqun: Undo the __preempt_count_{add,sub}() optimization in 32-bit preempt count since it may introduce {over,under}flow]
[boqun: Address the feedback from Shrikanth]
--- arch/x86/Kconfig | 1 + arch/x86/include/asm/preempt.h | 55 +++++++++++++------ arch/x86/kernel/cpu/common.c | 2 +- include/linux/hardirq.h | 47 +++++++++++----- include/linux/preempt.h | 23 +++++++- kernel/Kconfig.preempt | 4 ++ kernel/sched/core.c | 12 +++- kernel/softirq.c | 6 ++ .../testing/selftests/bpf/bpf_experimental.h | 2 +- 9 files changed, 115 insertions(+), 37 deletions(-)
arch/x86/Kconfig +1 -0
@@ -326,6 +326,7 @@ config X86 select USER_STACKTRACE_SUPPORT select HAVE_ARCH_KCSAN if X86_64 select PROC_PID_ARCH_STATUS if PROC_FS + select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16 select FUNCTION_ALIGNMENT_4B
arch/x86/include/asm/preempt.h +38 -17
123 lines not on this page · raw message
arch/x86/kernel/cpu/common.c +1 -1
@@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;EXPORT_PER_CPU_SYMBOL (current_task);EXPORT_PER_CPU_SYMBOL (const_current_task); -DEFINE_PER_CPU_CACHE_HOT (int, __preempt_count) =INIT_PREEMPT_COUNT ; +DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) = INIT_PREEMPT_COUNT;EXPORT_PER_CPU_SYMBOL (__preempt_count); DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
include/linux/hardirq.h +34 -13
78 lines not on this page · raw message
include/linux/preempt.h +20 -3
@@ -34,14 +34,31 @@* SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_DISABLE_MASK: 0x00ff0000 * HARDIRQ_MASK: 0x0f000000 +* +* When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a +* separate word and that allows 64bit load-store architectures to 'set' +* PREEMPT_NEED_RESCHED without messing up the otherwise symmetric +* modifications used on preempt_count and still load the whole thing +* (single-copy) atomically, without having to resort to full atomic +* operations. +* +* Because of the above, NMI_MASK bits are different depending on +* HAS_SEPARATE_PREEMPT_RESCHED_BITS: +* +* - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n: +* * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 +* +* - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y: +* NMI_MASK: 0xf0000000 +* (PREEMPT_NEED_RESCHED is in a different word) */ #define PREEMPT_BITS 8#define SOFTIRQ_BITS 8#define HARDIRQ_DISABLE_BITS 8#define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
@@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void)* preempt_count() is commonly implemented with READ_ONCE(). */ -#define nmi_count () (preempt_count() & NMI_MASK) -#define hardirq_count () (preempt_count() & HARDIRQ_MASK) +#define nmi_count () (preempt_count() & NMI_MASK) +#define hardirq_count () (preempt_count() & HARDIRQ_MASK)#ifdef CONFIG_PREEMPT_RT # define softirq_count () (current->softirq_disable_cnt & SOFTIRQ_MASK)# define irq_count () ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
kernel/Kconfig.preempt +4 -0
@@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK config PREEMPT_COUNT bool +config HAS_SEPARATE_PREEMPT_RESCHED_BITS + bool + depends on PREEMPT_COUNT && 64BIT + config PREEMPTION bool select PREEMPT_COUNT
kernel/sched/core.c +10 -2
@@ -5973,8 +5973,13 @@ void preempt_count_add(int val)#ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? +* +* Cannot detect underflow based on the current preempt_count() value +* if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32 +* bits. */ -if (DEBUG_LOCKS_WARN_ON ((preempt_count () <0 ))) +if (!IS_ENABLED (CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS ) && +DEBUG_LOCKS_WARN_ON ((preempt_count () <0 )))return ;#endif __preempt_count_add (val);
@@ -6006,7 +6011,10 @@ void preempt_count_sub(int val)/* * Underflow? */ -if (DEBUG_LOCKS_WARN_ON (val >preempt_count ())) +unsigned int uval = val; +unsigned int pc =preempt_count (); + +if (DEBUG_LOCKS_WARN_ON (pc - uval > pc))return ;/* * Is the spinlock portion underflowing?
kernel/softirq.c +6 -0
@@ -103,7 +103,13 @@ void _local_interrupt_enable(void) }EXPORT_SYMBOL (_local_interrupt_enable); +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +/* +* Any 32bit architecture that still cares about performance should +* probably ensure this is near preempt_count. +*/ DEFINE_PER_CPU(unsigned int, nmi_nesting); +#endif /* * SOFTIRQ_OFFSET usage:
tools/testing/selftests/bpf/bpf_experimental.h +1 -1
@@ -368,7 +368,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,#define SOFTIRQ_BITS 8#define HARDIRQ_DISABLE_BITS 8#define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 12/18 arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
In reply to Boqun FengArm64 already uses 64-bit preempt count and the need reschedule bit is maintained in a separate 32-bit word from the preempt count. Therefore preempt count has enough bits to represent 16 levels of NMI nesting, hence enable it for arm64. This saves a per-CPU variable and additional instructions in the NMI path.
--- arch/arm64/Kconfig | 1 + 1 file changed, 1 insertion(+)
arch/arm64/Kconfig +1 -0
@@ -247,6 +247,7 @@ config ARM64 select PCI_SYSCALL if PCI select POWER_RESET select POWER_SUPPLY + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 13/18 s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
In reply to Boqun FengFrom: Heiko Carstens <[email protected]>
Convert s390's preempt_count to 64 bit, and change the preempt primitives accordingly.
--- arch/s390/Kconfig | 1 + arch/s390/include/asm/lowcore.h | 13 +++++++--- arch/s390/include/asm/preempt.h | 43 +++++++++++++++------------------ 3 files changed, 30 insertions(+), 27 deletions(-)
arch/s390/Kconfig +1 -0
@@ -273,6 +273,7 @@ config S390 select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI select PCI_QUIRKS if PCI + select HAS_SEPARATE_PREEMPT_RESCHED_BITS select SPARSE_IRQ select SWIOTLB select SYSCTL_EXCEPTION_TRACE
arch/s390/include/asm/lowcore.h +9 -4
@@ -160,10 +160,15 @@ struct lowcore {/* SMP info area */ __u32 cpu_nr;/* 0x03a0 */ __u32 softirq_pending;/* 0x03a4 */ -__s32 preempt_count;/* 0x03a8 */ -__u32 spinlock_lockval;/* 0x03ac */ -__u32 spinlock_index;/* 0x03b0 */ -__u8 pad_0x03b4[0x03b8 -0x03b4 ];/* 0x03b4 */ +union { +struct { +__u32 need_resched;/* 0x03a8 */ +__u32 count;/* 0x03ac */ + } preempt; +__u64 preempt_count;/* 0x03a8 */ + }; +__u32 spinlock_lockval;/* 0x03b0 */ +__u32 spinlock_index;/* 0x03b4 */ __u64 percpu_offset;/* 0x03b8 */ __u8 percpu_register;/* 0x03c0 */ __u8 pad_0x03c1[0x0400 -0x03c1 ];/* 0x03c1 */
arch/s390/include/asm/preempt.h +20 -23
105 lines not on this page · raw message
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH v5 14/18 rust: Introduce interrupt module
In reply to Boqun FengFrom: Lyude Paul <[email protected]>
This introduces a module for dealing with interrupt-disabled contexts, including the ability to enable and disable interrupts along with the ability to annotate functions as expecting that IRQs are already disabled on the local CPU.
--- rust/helpers/helpers.c | 1 + rust/helpers/interrupt.c | 18 ++++++++ rust/helpers/sync.c | 5 +++ rust/kernel/interrupt.rs | 89 ++++++++++++++++++++++++++++++++++++++++ rust/kernel/lib.rs | 1 + 5 files changed, 114 insertions(+) create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/kernel/interrupt.rs
rust/helpers/helpers.c +1 -0
@@ -65,6 +65,7 @@#include "irq.c" #include "fs.c" #include "gpu.c" +#include "interrupt.c" #include "io.c" #include "jump_label.c" #include "kunit.c"
rust/helpers/interrupt.c new file +18 -0
@@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/spinlock.h> + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ +local_interrupt_disable (); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ +local_interrupt_enable (); +} + +__rust_helper bool rust_helper_irqs_disabled(void) +{ +return irqs_disabled (); +}
rust/helpers/sync.c +5 -0
@@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) {lockdep_unregister_key (k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ +lockdep_assert_irqs_disabled (); +}
rust/kernel/interrupt.rs new file +89 -0
89 lines not on this page · raw message
rust/kernel/lib.rs +1 -0
@@ -82,6 +82,7 @@pub mod impl_flags;pub mod init;pub mod interop; +pub mod interrupt;pub mod io;pub mod ioctl;pub mod iommu;
Signature · 1 line
2.50.1 (Apple Git-155)
Peter Zijlstra
maintainerRe: [PATCH v5 14/18] rust: Introduce interrupt module
Quoted 14 lines · Boqun Feng
rust/helpers/interrupt.c new file +8
-0@@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/spinlock.h> + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ +local_interrupt_disable (); +}
rust/helpers/interrupt.c line 18
Quoted 10 lines
rust/helpers/interrupt.c new file +10
-0+ +__rust_helper void rust_helper_local_interrupt_enable(void) +{ +local_interrupt_enable (); +} + +__rust_helper bool rust_helper_irqs_disabled(void) +{ +return irqs_disabled (); +}
This last one doesn't appear used in this patch; can I remove it?
Boqun Feng
maintainerRe: [PATCH v5 14/18] rust: Introduce interrupt module
Quoted 19 lines · Peter Zijlstra
Quoted 18 lines · Boqun Feng
rust/helpers/interrupt.c new file +12
-0@@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/spinlock.h> + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ +local_interrupt_disable (); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ +local_interrupt_enable ();
Quoted 8 lines
rust/helpers/interrupt.c line 18
Quoted 6 lines
rust/helpers/interrupt.c new file +6
-0+} + +__rust_helper bool rust_helper_irqs_disabled(void) +{ +return irqs_disabled (); +}This last one doesn't appear used in this patch; can I remove it?
Yes. Previously this function was used for debugging, but we have moved to lockdep_assert_irqs_disabled(), so it's no longer used.
Regards,
Boqun
[tip: locking/core] rust: Introduce interrupt module
In reply to Boqun FengThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: c1f0451ec748f7a58a4e50bcb8f75bd89718be78 Gitweb: https://git.kernel.org/tip/c1f0451ec748f7a58a4e50bcb8f75bd89718be78 Author: Lyude Paul <[email protected]> AuthorDate: Fri, 07 Aug 2026 00:02:11 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 10 Aug 2026 12:53:09 +02:00
rust: Introduce interrupt module
This introduces a module for dealing with interrupt-disabled contexts, including the ability to enable and disable interrupts along with the ability to annotate functions as expecting that IRQs are already disabled on the local CPU.
--- rust/helpers/helpers.c | 1 +- rust/helpers/interrupt.c | 13 ++++++- rust/helpers/sync.c | 5 ++- rust/kernel/interrupt.rs | 89 +++++++++++++++++++++++++++++++++++++++- rust/kernel/lib.rs | 1 +- 5 files changed, 109 insertions(+) create mode 100644 rust/helpers/interrupt.c create mode 100644 rust/kernel/interrupt.rs
rust/helpers/helpers.c +1 -0
@@ -65,6 +65,7 @@#include "irq.c" #include "fs.c" #include "gpu.c" +#include "interrupt.c" #include "io.c" #include "jump_label.c" #include "kunit.c"
rust/helpers/interrupt.c new file +13 -0
@@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/spinlock.h> + +__rust_helper void rust_helper_local_interrupt_disable(void) +{ +local_interrupt_disable (); +} + +__rust_helper void rust_helper_local_interrupt_enable(void) +{ +local_interrupt_enable (); +}
rust/helpers/sync.c +5 -0
@@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) {lockdep_unregister_key (k); } + +__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void) +{ +lockdep_assert_irqs_disabled (); +}
rust/kernel/interrupt.rs new file +89 -0
89 lines not on this page · raw message
rust/kernel/lib.rs +1 -0
@@ -82,6 +82,7 @@ pub mod id_pool;pub mod impl_flags;pub mod init;pub mod interop; +pub mod interrupt;pub mod io;pub mod ioctl;pub mod iommu;
Boqun Feng
maintainerPATCH v5 15/18 rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers
In reply to Boqun Fengspin_lock_irq_disable() and spin_unlock_irq_enable() are inline functions, to use them in Rust helpers are introduced. This is for interrupt disabling lock abstraction in Rust.
--- rust/helpers/spinlock.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
rust/helpers/spinlock.c +15 -0
@@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) {lockdep_assert_held (lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ +spin_lock_irq_disable (lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ +spin_unlock_irq_enable (lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ +return spin_trylock_irq_disable (lock); +}
Signature · 1 line
2.50.1 (Apple Git-155)
[tip: locking/core] rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 78c998b04ef53492f3502051349e55615681e00f Gitweb: https://git.kernel.org/tip/78c998b04ef53492f3502051349e55615681e00f Author: Boqun Feng <[email protected]> AuthorDate: Fri, 07 Aug 2026 00:02:12 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 10 Aug 2026 12:53:10 +02:00
rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers
spin_lock_irq_disable() and spin_unlock_irq_enable() are inline functions, to use them in Rust helpers are introduced. This is for interrupt disabling lock abstraction in Rust.
--- rust/helpers/spinlock.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+)
rust/helpers/spinlock.c +15 -0
@@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) {lockdep_assert_held (lock); } + +__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock) +{ +spin_lock_irq_disable (lock); +} + +__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock) +{ +spin_unlock_irq_enable (lock); +} + +__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock) +{ +return spin_trylock_irq_disable (lock); +}
Boqun Feng
maintainerPATCH v5 16/18 rust: sync: Use super::* in spinlock.rs
In reply to Boqun FengFrom: Lyude Paul <[email protected]>
No functional changes.
--- rust/kernel/sync/lock/spinlock.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)
rust/kernel/sync/lock/spinlock.rs +4 -5
@@ -3,6 +3,7 @@//! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super ::*;/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. ///
@@ -82,7 +83,7 @@ macro_rules! new_spinlock {/// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock <T > =super ::Lock <T ,SpinLockBackend >; +pub type SpinLock <T > =Lock <T ,SpinLockBackend >;/// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend ;
@@ -91,13 +92,11 @@ macro_rules! new_spinlock {/// /// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard <'a,T > =super ::Guard <'a,T ,SpinLockBackend >; +pub type SpinLockGuard <'a,T > =Guard <'a,T ,SpinLockBackend >;// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super ::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend {type State = bindings::spinlock_t ;type GuardState = ();
Signature · 1 line
2.50.1 (Apple Git-155)
[tip: locking/core] rust: sync: Use super::* in spinlock.rs
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: df9165ab5254acb2035919c6e0511de799c1e8c1 Gitweb: https://git.kernel.org/tip/df9165ab5254acb2035919c6e0511de799c1e8c1 Author: Lyude Paul <[email protected]> AuthorDate: Fri, 07 Aug 2026 00:02:13 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 10 Aug 2026 12:53:10 +02:00
rust: sync: Use super::* in spinlock.rs
No functional changes.
--- rust/kernel/sync/lock/spinlock.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-)
rust/kernel/sync/lock/spinlock.rs +4 -5
@@ -3,6 +3,7 @@//! A kernel spinlock. //! //! This module allows Rust code to use the kernel's `spinlock_t`. +use super ::*;/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. ///
@@ -82,7 +83,7 @@ pub use new_spinlock;/// ``` /// /// [`spinlock_t`]: srctree/include/linux/spinlock.h -pub type SpinLock <T > =super ::Lock <T ,SpinLockBackend >; +pub type SpinLock <T > =Lock <T ,SpinLockBackend >;/// A kernel `spinlock_t` lock backend. pub struct SpinLockBackend ;
@@ -91,13 +92,11 @@ pub struct SpinLockBackend;/// /// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock /// the [`SpinLock`] upon being dropped. -/// -/// [`Guard`]: super::Guard -pub type SpinLockGuard <'a,T > =super ::Guard <'a,T ,SpinLockBackend >; +pub type SpinLockGuard <'a,T > =Guard <'a,T ,SpinLockBackend >;// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the // default implementation that always calls the same locking method. -unsafe impl super ::Backend for SpinLockBackend { +unsafe impl Backend for SpinLockBackend {type State = bindings::spinlock_t ;type GuardState = ();
Boqun Feng
maintainerPATCH v5 17/18 rust: sync: Add SpinLockIrq
In reply to Boqun FengFrom: Lyude Paul <[email protected]>
A variant of SpinLock that ensures interrupts are disabled in the critical section. lock() will ensure that either interrupts are already disabled or disable them. unlock() will reverse the respective operation.
[Boqun: Port to use spin_lock_irq_disable() andspin_unlock_irq_enable()]
--- rust/kernel/sync.rs | 9 +- rust/kernel/sync/lock/global.rs | 3 + rust/kernel/sync/lock/spinlock.rs | 230 ++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+), 1 deletion(-)
rust/kernel/sync.rs +8 -1
@@ -27,7 +27,14 @@pub use condvar::{new_condvar,CondVar ,CondVarTimeoutResult };pub use lock::global::{global_lock,GlobalGuard ,GlobalLock ,GlobalLockBackend ,GlobalLockedBy };pub use lock::mutex::{new_mutex,Mutex ,MutexGuard }; -pub use lock::spinlock::{new_spinlock,SpinLock ,SpinLockGuard }; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, +SpinLock , +SpinLockGuard , +SpinLockIrq , +SpinLockIrqGuard ,// +};pub use locked_by::LockedBy ;pub use refcount::Refcount ;pub use set_once::SetOnce ;
rust/kernel/sync/lock/global.rs +3 -0
@@ -306,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) => { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) => { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; }
rust/kernel/sync/lock/spinlock.rs +230 -0
239 lines not on this page · raw message
Signature · 1 line
2.50.1 (Apple Git-155)
Peter Zijlstra
maintainerRe: [PATCH v5 17/18] rust: sync: Add SpinLockIrq
rust/kernel/sync.rs line 37
Quoted 9 lines · Boqun Feng
rust/kernel/sync.rs +8
-1-pub use lock::spinlock::{new_spinlock,SpinLock ,SpinLockGuard }; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, +SpinLock , +SpinLockGuard , +SpinLockIrq , +SpinLockIrqGuard ,// +};
What's this new_spinlock* stuff about? It has sensible naming; unlike that CameCase nonsense the rest has.
Boqun Feng
maintainerRe: [PATCH v5 17/18] rust: sync: Add SpinLockIrq
Quoted 14 lines · Peter Zijlstra
rust/kernel/sync.rs line 37
Quoted 9 lines · Boqun Feng
rust/kernel/sync.rs +8
-1-pub use lock::spinlock::{new_spinlock,SpinLock ,SpinLockGuard }; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, +SpinLock , +SpinLockGuard , +SpinLockIrq , +SpinLockIrqGuard ,// +};What's this new_spinlock* stuff about? It has sensible naming; unlike that CameCase nonsense the rest has.
They're the counter part of *spin_lock_init() in C side, similar as C, they need to macros to expand at callsite for lock class key generation. Types and Traits use CameCase, and macros and functions use snake_case.
Regards,
Boqun
[tip: locking/core] rust: sync: Add SpinLockIrq
In reply to Boqun FengThe following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 5967f4df55521de596cbe34e0c0c96e962ef3f2e Gitweb: https://git.kernel.org/tip/5967f4df55521de596cbe34e0c0c96e962ef3f2e Author: Lyude Paul <[email protected]> AuthorDate: Fri, 07 Aug 2026 00:02:14 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 10 Aug 2026 12:53:10 +02:00
rust: sync: Add SpinLockIrq
A variant of SpinLock that ensures interrupts are disabled in the critical section. lock() will ensure that either interrupts are already disabled or disable them. unlock() will reverse the respective operation.
[Boqun: Port to use spin_lock_irq_disable() andspin_unlock_irq_enable()]
--- rust/kernel/sync.rs | 9 +- rust/kernel/sync/lock/global.rs | 3 +- rust/kernel/sync/lock/spinlock.rs | 230 +++++++++++++++++++++++++++++- 3 files changed, 241 insertions(+), 1 deletion(-)
rust/kernel/sync.rs +8 -1
@@ -27,7 +27,14 @@ pub use completion::Completion;pub use condvar::{new_condvar,CondVar ,CondVarTimeoutResult };pub use lock::global::{global_lock,GlobalGuard ,GlobalLock ,GlobalLockBackend ,GlobalLockedBy };pub use lock::mutex::{new_mutex,Mutex ,MutexGuard }; -pub use lock::spinlock::{new_spinlock,SpinLock ,SpinLockGuard }; +pub use lock::spinlock::{ + new_spinlock, + new_spinlock_irq, +SpinLock , +SpinLockGuard , +SpinLockIrq , +SpinLockIrqGuard ,// +};pub use locked_by::LockedBy ;pub use refcount::Refcount ;pub use set_once::SetOnce ;
rust/kernel/sync/lock/global.rs +3 -0
@@ -306,4 +306,7 @@ macro_rules! global_lock_inner { (backend SpinLock) => { $crate::sync::lock::spinlock::SpinLockBackend }; + (backend SpinLockIrq) => { + $crate::sync::lock::spinlock::SpinLockIrqBackend + }; }
rust/kernel/sync/lock/spinlock.rs +230 -0
239 lines not on this page · raw message
Boqun Feng
maintainerPATCH v5 18/18 rust: sync: Introduce SpinLockIrq::lock_with() and friends
In reply to Boqun FengFrom: Lyude Paul <[email protected]>
SpinLockIrq and SpinLock use the exact same underlying C structure, with the only real difference being that the former uses the irq_disable() and irq_enable() variants for locking/unlocking. These variants can introduce some minor overhead in contexts where we already know that local processor interrupts are disabled, and as such we want a way to be able to skip modifying processor interrupt state in said contexts in order to avoid some overhead - just like the current C API allows us to do.
In order to do this, we add some special functions for SpinLockIrq:lock_with() and try_lock_with(), which allow acquiring the lock without changing the interrupt state - as long as the caller can provide a LocalInterruptDisabled reference to prove that local processor interrupts have been disabled.
In some hacked-together benchmarks we ran, most of the time this did actually seem to lead to a noticeable difference in overhead:
From an aarch64 VM running on a MacBook M4:
lock() when irq is disabled, 100 times cost Delta { nanos: 500 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 292 }
lock() when irq is enabled, 100 times cost Delta { nanos: 834 }
lock() when irq is disabled, 100 times cost Delta { nanos: 459 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 291 }
lock() when irq is enabled, 100 times cost Delta { nanos: 709 } From an x86_64 VM (qemu/kvm) running on a i7-13700H
lock() when irq is disabled, 100 times cost Delta { nanos: 1002 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 729 }
lock() when irq is enabled, 100 times cost Delta { nanos: 1516 }
lock() when irq is disabled, 100 times cost Delta { nanos: 754 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 966 }
lock() when irq is enabled, 100 times cost Delta { nanos: 1227 }
(note that there were some runs on x86_64 where lock() on irq
disabled vs. lock_with() on irq disabled had equivalent benchmarks,
but it very much appeared to be a minority of test runs.)While it's not clear how this affects real-world workloads yet, let's add this for the time being so we can find out.
This makes it so that a SpinLockIrq will work like a SpinLock if interrupts are disabled. So a function:
(&'a SpinLockIrq, &'a LocalInterruptDisabled) -> Guard<'a, .., SpinLockBackend>
makes sense. Note that due to Guard and LocalInterruptDisabled having the same lifetime, interrupts cannot be enabled while the Guard exists.
--- rust/kernel/sync/lock/spinlock.rs | 92 ++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-)
rust/kernel/sync/lock/spinlock.rs +91 -1
@@ -4,7 +4,10 @@//! //! This module allows Rust code to use the kernel's `spinlock_t`. use super ::*; -use crate ::prelude::*; +use crate ::{ + interrupt::LocalInterruptDisabled , + prelude::*,// +};/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. ///
@@ -160,6 +163,16 @@ macro_rules! new_spinlock_irq {/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. /// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the +/// interrupt state to ensure that local processor interrupts remain disabled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and +/// local processor interrupts are already known to be disabled, in which case the local +/// interrupt state will not be touched. This method should be preferred if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// /// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, /// [see the interrupt module](kernel::interrupt). ///
@@ -213,7 +226,47 @@ macro_rules! new_spinlock_irq {/// # Ok::<(), Error>(()) /// ``` /// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function +/// which can only be called when local processor interrupts are already disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq<Inner>, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) { +/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard = local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// /// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with pub type SpinLockIrq <T > =super ::Lock <T ,SpinLockIrqBackend >;/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
@@ -278,6 +331,43 @@ unsafe fn assert_is_held(ptr: *mut Self::State) { } } +impl <T : ?Sized >Lock <T ,SpinLockIrqBackend > { +/// Casts the lock as a `Lock<T, SpinLockBackend>`. +#[inline] +fn as_lock_in_interrupt <'a>(&'a self, _context: &'aLocalInterruptDisabled ) -> &'aSpinLock <T > { +// SAFETY: +// - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data +// layouts. +// - As long as local interrupts are disabled (which is proven to be true by _context), it +// is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock. +unsafe { core::mem::transmute (self) } + } + +/// Acquires the lock without modifying local interrupt state. +/// +/// This function should be used in place of the more expensive [`Lock::lock()`] function when +/// possible for [`SpinLockIrq`] locks. +#[inline] +pub fn lock_with <'a>(&'a self, context: &'aLocalInterruptDisabled ) ->SpinLockGuard <'a,T > { + self.as_lock_in_interrupt (context).lock () + } + +/// Tries to acquire the lock without modifying local interrupt state. +/// +/// This function should be used in place of the more expensive [`Lock::try_lock()`] function +/// when possible for [`SpinLockIrq`] locks. +/// +/// Returns a guard that can be used to access the data protected by the lock if successful. +#[must_use = "if unused, the lock will be immediately unlocked" ] +#[inline] +pub fn try_lock_with <'a>( + &'a self, + context: &'aLocalInterruptDisabled , + ) ->Option <SpinLockGuard <'a,T >> { + self.as_lock_in_interrupt (context).try_lock () + } +} +#[kunit_tests(rust_spinlock_irq_condvar)] mod tests {use super ::*;
Signature · 1 line
2.50.1 (Apple Git-155)
[tip: locking/core] rust: sync: Introduce SpinLockIrq::lock_with() and friends
The following commit has been merged into the locking/core branch of tip:
The following commit has been merged into the locking/core branch of tip:
Commit-ID: ec4fad7c2bdc80c62fa73f799b77ad299f73dcc3 Gitweb: https://git.kernel.org/tip/ec4fad7c2bdc80c62fa73f799b77ad299f73dcc3 Author: Lyude Paul <[email protected]> AuthorDate: Fri, 07 Aug 2026 00:02:15 -07:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Mon, 10 Aug 2026 12:53:11 +02:00
rust: sync: Introduce SpinLockIrq::lock_with() and friends
SpinLockIrq and SpinLock use the exact same underlying C structure, with the only real difference being that the former uses the irq_disable() and irq_enable() variants for locking/unlocking. These variants can introduce some minor overhead in contexts where we already know that local processor interrupts are disabled, and as such we want a way to be able to skip modifying processor interrupt state in said contexts in order to avoid some overhead - just like the current C API allows us to do.
In order to do this, we add some special functions for SpinLockIrq:lock_with() and try_lock_with(), which allow acquiring the lock without changing the interrupt state - as long as the caller can provide a LocalInterruptDisabled reference to prove that local processor interrupts have been disabled.
In some hacked-together benchmarks we ran, most of the time this did actually seem to lead to a noticeable difference in overhead:
From an aarch64 VM running on a MacBook M4:
lock() when irq is disabled, 100 times cost Delta { nanos: 500 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 292 }
lock() when irq is enabled, 100 times cost Delta { nanos: 834 }
lock() when irq is disabled, 100 times cost Delta { nanos: 459 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 291 }
lock() when irq is enabled, 100 times cost Delta { nanos: 709 } From an x86_64 VM (qemu/kvm) running on a i7-13700H
lock() when irq is disabled, 100 times cost Delta { nanos: 1002 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 729 }
lock() when irq is enabled, 100 times cost Delta { nanos: 1516 }
lock() when irq is disabled, 100 times cost Delta { nanos: 754 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 966 }
lock() when irq is enabled, 100 times cost Delta { nanos: 1227 }
(note that there were some runs on x86_64 where lock() on irq
disabled vs. lock_with() on irq disabled had equivalent benchmarks,
but it very much appeared to be a minority of test runs.)While it's not clear how this affects real-world workloads yet, let's add this for the time being so we can find out.
This makes it so that a SpinLockIrq will work like a SpinLock if interrupts are disabled. So a function:
(&'a SpinLockIrq, &'a LocalInterruptDisabled) -> Guard<'a, .., SpinLockBackend>
makes sense. Note that due to Guard and LocalInterruptDisabled having the same lifetime, interrupts cannot be enabled while the Guard exists.
--- rust/kernel/sync/lock/spinlock.rs | 92 +++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-)
rust/kernel/sync/lock/spinlock.rs +91 -1
@@ -4,7 +4,10 @@//! //! This module allows Rust code to use the kernel's `spinlock_t`. use super ::*; -use crate ::prelude::*; +use crate ::{ + interrupt::LocalInterruptDisabled , + prelude::*,// +};/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class. ///
@@ -160,6 +163,16 @@ pub use new_spinlock_irq;/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section. /// +/// This lock can be acquired in two ways: +/// +/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the +/// interrupt state to ensure that local processor interrupts remain disabled for at least as +/// long as the [`SpinLockIrqGuard`] exists. +/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and +/// local processor interrupts are already known to be disabled, in which case the local +/// interrupt state will not be touched. This method should be preferred if a +/// [`LocalInterruptDisabled`] token is present in the scope. +/// /// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts, /// [see the interrupt module](kernel::interrupt). ///
@@ -213,7 +226,47 @@ pub use new_spinlock_irq;/// # Ok::<(), Error>(()) /// ``` /// +/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function +/// which can only be called when local processor interrupts are already disabled. +/// +/// ``` +/// use kernel::sync::{new_spinlock_irq, SpinLockIrq}; +/// use kernel::interrupt::*; +/// +/// struct Inner { +/// a: u32, +/// } +/// +/// #[pin_data] +/// struct Example { +/// #[pin] +/// inner: SpinLockIrq<Inner>, +/// } +/// +/// impl Example { +/// fn new() -> impl PinInit<Self> { +/// pin_init!(Self { +/// inner <- new_spinlock_irq!(Inner { a: 20 }), +/// }) +/// } +/// } +/// +/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts. +/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) { +/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling +/// // interrupt state using lock_with() and the provided token +/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20); +/// } +/// +/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?; +/// # let interrupt_guard = local_interrupt_disable(); +/// # noirq_work(&e, &interrupt_guard); +/// # +/// # Ok::<(), Error>(()) +/// ``` +/// /// [`lock()`]: SpinLockIrq::lock +/// [`lock_with()`]: SpinLockIrq::lock_with pub type SpinLockIrq <T > =super ::Lock <T ,SpinLockIrqBackend >;/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
@@ -278,6 +331,43 @@ unsafe impl Backend for SpinLockIrqBackend { } } +impl <T : ?Sized >Lock <T ,SpinLockIrqBackend > { +/// Casts the lock as a `Lock<T, SpinLockBackend>`. +#[inline] +fn as_lock_in_interrupt <'a>(&'a self, _context: &'aLocalInterruptDisabled ) -> &'aSpinLock <T > { +// SAFETY: +// - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data +// layouts. +// - As long as local interrupts are disabled (which is proven to be true by _context), it +// is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock. +unsafe { core::mem::transmute (self) } + } + +/// Acquires the lock without modifying local interrupt state. +/// +/// This function should be used in place of the more expensive [`Lock::lock()`] function when +/// possible for [`SpinLockIrq`] locks. +#[inline] +pub fn lock_with <'a>(&'a self, context: &'aLocalInterruptDisabled ) ->SpinLockGuard <'a,T > { + self.as_lock_in_interrupt (context).lock () + } + +/// Tries to acquire the lock without modifying local interrupt state. +/// +/// This function should be used in place of the more expensive [`Lock::try_lock()`] function +/// when possible for [`SpinLockIrq`] locks. +/// +/// Returns a guard that can be used to access the data protected by the lock if successful. +#[must_use = "if unused, the lock will be immediately unlocked" ] +#[inline] +pub fn try_lock_with <'a>( + &'a self, + context: &'aLocalInterruptDisabled , + ) ->Option <SpinLockGuard <'a,T >> { + self.as_lock_in_interrupt (context).try_lock () + } +} +#[kunit_tests(rust_spinlock_irq_condvar)] mod tests {use super ::*;
Peter Zijlstra
maintainerQuoted 4 lines · Boqun Feng
* Added underflow/overflow checking for interrupt disabling counter as suggested by Shrikanth.
* Added a max nesting level per suggestion from Shrikanth.
* Document/comment updates.
Can I get those as separate patches? I had already queued and tested the previous version.
(although as it turns out, I lost one SoB fixup somehow, so I'm applying them again with just that Changelog ammended)
Peter Zijlstra
maintainerQuoted 7 lines · Peter Zijlstra
Quoted 4 lines · Boqun Feng
* Added underflow/overflow checking for interrupt disabling counter as suggested by Shrikanth.
* Added a max nesting level per suggestion from Shrikanth.
* Document/comment updates.
Quoted 5 lines
Can I get those as separate patches? I had already queued and tested the previous version.
(although as it turns out, I lost one SoB fixup somehow, so I'm applying them again with just that Changelog ammended)
I'll grab the rust patches from this set I suppose. I had not yet looked at those, since I don't normally build rust crud and I need to figure out how to do that again :-(
Peter Zijlstra
maintainerQuoted 11 lines · Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
Quoted 4 lines · Boqun Feng
* Added underflow/overflow checking for interrupt disabling counter as suggested by Shrikanth.
* Added a max nesting level per suggestion from Shrikanth.
* Document/comment updates.Can I get those as separate patches? I had already queued and tested the previous version.
Quoted 6 lines
Quoted 2 lines
(although as it turns out, I lost one SoB fixup somehow, so I'm applying them again with just that Changelog ammended)
I'll grab the rust patches from this set I suppose. I had not yet looked at those, since I don't normally build rust crud and I need to figure out how to do that again :-(
So on tip/master + tip/locking/core (its fresh, so not in master yet).
I have debian experimental rustc, which is build against clang-23, so I do:
$ make O=defconfig-build/ LLVM=-23 defconfig rust.config $ make O=defconfig-build/ LLVM=-23 -j64
and that gets me a wild amount of build fail :/
error[E0560]: struct blk_mq_tag_set has no field named ops
--> ../rust/kernel/block/mq/tag_set.rs:46:21
| 46 | ops: OperationsVTable::<T>::build(), | ^^^ `blk_mq_tag_set` does not have this field |
= note: available fields are: _address
error[E0560]: struct blk_mq_tag_set has no field named nr_hw_queues
--> ../rust/kernel/block/mq/tag_set.rs:47:21
| 47 | nr_hw_queues, | ^^^^^^^^^^^^ `blk_mq_tag_set` does not have this field |
= note: available fields are: _address
... this goes on for a fair while.
And this is *before* I've applied any of the rust patches from this series.
Why can't stuff just ever work :-(
Gary Guo
maintainerQuoted 39 lines · Peter Zijlstra
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
Quoted 4 lines · Boqun Feng
* Added underflow/overflow checking for interrupt disabling counter as suggested by Shrikanth.
* Added a max nesting level per suggestion from Shrikanth.
* Document/comment updates.Can I get those as separate patches? I had already queued and tested the previous version.
Quoted 6 lines
Quoted 2 lines
(although as it turns out, I lost one SoB fixup somehow, so I'm applying them again with just that Changelog ammended)
I'll grab the rust patches from this set I suppose. I had not yet looked at those, since I don't normally build rust crud and I need to figure out how to do that again :-(
So on tip/master + tip/locking/core (its fresh, so not in master yet).
I have debian experimental rustc, which is build against clang-23, so I do:
$ make O=defconfig-build/ LLVM=-23 defconfig rust.config $ make O=defconfig-build/ LLVM=-23 -j64and that gets me a wild amount of build fail :/
error[E0560]: struct
blk_mq_tag_sethas no field namedops
-->../rust/kernel/block/mq/tag_set.rs:46:21| 46 | ops: OperationsVTable::<T>::build(), | ^^^ `blk_mq_tag_set` does not have this field |= note: available fields are:
_addresserror[E0560]: struct
blk_mq_tag_sethas no field namednr_hw_queues
-->../rust/kernel/block/mq/tag_set.rs:47:21
Quoted 7 lines
| 47 | nr_hw_queues, | ^^^^^^^^^^^^ `blk_mq_tag_set` does not have this field |= note: available fields are:
_address... this goes on for a fair while.
There was a change in clang 22 that breaks bindgen. You'd need the latest bindgen, see
https://lore.kernel.org/rust-for-linux/CABwQupNfMAJOGqRM9ke6tj4f53dCCsBDKU7Vp+zf8mwk7bqt8Q@mail.gmail.com/.
If you're using debian unstable clang/rustc then you should also get bindgen from there too. Debian sid has the bindgen version that works with latest clang.
Best,
Gary
Peter Zijlstra
maintainerQuoted 46 lines · Gary Guo
Quoted 45 lines · Peter Zijlstra
Quoted 11 lines · Peter Zijlstra
Quoted 10 lines · Peter Zijlstra
Quoted 4 lines · Boqun Feng
* Added underflow/overflow checking for interrupt disabling counter as suggested by Shrikanth.
* Added a max nesting level per suggestion from Shrikanth.
* Document/comment updates.Can I get those as separate patches? I had already queued and tested the previous version.
Quoted 6 lines
Quoted 2 lines
(although as it turns out, I lost one SoB fixup somehow, so I'm applying them again with just that Changelog ammended)
I'll grab the rust patches from this set I suppose. I had not yet looked at those, since I don't normally build rust crud and I need to figure out how to do that again :-(
So on tip/master + tip/locking/core (its fresh, so not in master yet).
I have debian experimental rustc, which is build against clang-23, so I do:
$ make O=defconfig-build/ LLVM=-23 defconfig rust.config $ make O=defconfig-build/ LLVM=-23 -j64and that gets me a wild amount of build fail :/
error[E0560]: struct
blk_mq_tag_sethas no field namedops
-->../rust/kernel/block/mq/tag_set.rs:46:21| 46 | ops: OperationsVTable::<T>::build(), | ^^^ `blk_mq_tag_set` does not have this field |= note: available fields are:
_addresserror[E0560]: struct
blk_mq_tag_sethas no field namednr_hw_queues
-->../rust/kernel/block/mq/tag_set.rs:47:21| 47 | nr_hw_queues, | ^^^^^^^^^^^^ `blk_mq_tag_set` does not have this field |= note: available fields are:
_address
Quoted 8 lines
Quoted 1 line
... this goes on for a fair while.
There was a change in clang 22 that breaks bindgen. You'd need the latest bindgen, see
https://lore.kernel.org/rust-for-linux/CABwQupNfMAJOGqRM9ke6tj4f53dCCsBDKU7Vp+zf8mwk7bqt8Q@mail.gmail.com/.If you're using debian unstable clang/rustc then you should also get bindgen from there too. Debian sid has the bindgen version that works with latest clang.
I 'upgraded' to bindgen/unstable, and found that Kbuild does not have a dependency on the bindgen version so it didn't rebuild and insta exploded with the same fails. A make clean fixed that.
Build seems happier now. Let me go apply these here locking patches.
Boqun Feng
maintainercover Improvements on SpinLockIrq patch series
In reply to Peter ZijlstraHi Peter,
Per your request, I send those improvements as separate patches. I did a very light build and unit tests. I'm still waiting for the 0day build test report in case we missed something subtle, but figured maybe it's better if I send them out while I'm waiting.
Regards,
Boqun
Boqun Feng (4):
irq: Add {over,under}flow detection for
local_interrupt_{enable,disable} selftests/bpf: Use the new NMI_BITS definition
irq: Explain better on NMI_MASK overflow condition
irq: Add max local_interrupt_disable() nesting level kunit test case
include/linux/hardirq.h | 2 +- include/linux/interrupt_rc.h | 13 +++++++++++ kernel/irq/refcount_interrupt_test.c | 22 +++++++++++++++++++ .../testing/selftests/bpf/bpf_experimental.h | 2 +- 4 files changed, 37 insertions(+), 2 deletions(-)
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH 1/4 irq: Add {over,under}flow detection for local_interrupt_{enable,disable}
Currently there is no way to detect overflow and underflow inside the HARDIRQ_DISABLE_MASK part of preempt_count. Add these detection guarded by DEBUG_PREEMPT similar to the detection for PREEMPT_MASK {over,under}flow.
--- include/linux/interrupt_rc.h | 13 +++++++++++++ 1 file changed, 13 insertions(+)
include/linux/interrupt_rc.h +13 -0
@@ -11,6 +11,7 @@* make it easier to include without hitting circular header dependencies. */ +#include <linux/debug_locks.h> #include <linux/irqflags.h> #include <linux/preempt.h> #include <linux/processor.h>
@@ -63,6 +64,13 @@ static inline void local_interrupt_disable(void) new_count =hardirq_disable_enter (); +/* Is hardirq disable count overflow soon? */ +if (IS_ENABLED (CONFIG_DEBUG_PREEMPT ) && +DEBUG_LOCKS_WARN_ON (((new_count &HARDIRQ_DISABLE_MASK ) + + (10 <<HARDIRQ_DISABLE_SHIFT )) >= +HARDIRQ_DISABLE_MASK )) +return ; +/* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */ if ((new_count &HARDIRQ_DISABLE_MASK ) ==HARDIRQ_DISABLE_OFFSET )
@@ -73,6 +81,11 @@ static inline void local_interrupt_enable(void) {int new_count; +/* Unpaired local_interrupt_enable()? Warn and abort. */ +if (IS_ENABLED (CONFIG_DEBUG_PREEMPT ) && +DEBUG_LOCKS_WARN_ON ((preempt_count () &HARDIRQ_DISABLE_MASK ) ==0 )) +return ; + new_count =hardirq_disable_exit ();if ((new_count &HARDIRQ_DISABLE_MASK ) ==0 )
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH 2/4 selftests/bpf: Use the new NMI_BITS definition
In reply to Boqun FengThe definition of NMI_BITS was out of sync after the introduction of HAS_SEPARATE_PREEMPT_RESCHED_BITS, hence adjust it.
--- tools/testing/selftests/bpf/bpf_experimental.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
tools/testing/selftests/bpf/bpf_experimental.h +1 -1
@@ -368,7 +368,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,#define SOFTIRQ_BITS 8#define HARDIRQ_DISABLE_BITS 8#define HARDIRQ_BITS 4 -#define NMI_BITS 1 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))#define PREEMPT_SHIFT 0#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH 3/4 irq: Explain better on NMI_MASK overflow condition
In reply to Boqun FengCurrently __preempt_count_nmi_enter() uses 15 as the maximum NMI nesting level, and it's because 4 bits are used to represent the nesting level of NMI. Replace the tautological comment with a more direct mention to the 4 bits limitation to help readers understand this condition.
--- include/linux/hardirq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/hardirq.h +1 -1
@@ -108,7 +108,7 @@ DECLARE_PER_CPU(unsigned int, nmi_nesting);#define __preempt_count_nmi_enter () \ do { \ __preempt_count_add(HARDIRQ_OFFSET); \ -/* Maximum NMI nesting is 15. */ \ +/* NMI nesting is represented in 4 bits. */ \BUG_ON (__this_cpu_read (nmi_nesting) >=15 ); \__this_cpu_inc (nmi_nesting); \preempt_count_set (preempt_count () |NMI_MASK ); \
Signature · 1 line
2.50.1 (Apple Git-155)
Boqun Feng
maintainerPATCH 4/4 irq: Add max local_interrupt_disable() nesting level kunit test case
In reply to Boqun FengTo confirm the max nesting level of local_interrupt_disable() works, a kunit test is added to the whole test suite.
Note that when DEBUG_PREEMPT=y, it'll generate a warning which is expected.
--- kernel/irq/refcount_interrupt_test.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+)
kernel/irq/refcount_interrupt_test.c +22 -0
@@ -52,6 +52,27 @@ static void test_multiple_irq_change(struct kunit *test)TEST_IRQ_ON (); } +static void test_max_nesting_irq_change (struct kunit *test) +{ +for (int i =0 ; i <__IRQ_MASK (HARDIRQ_DISABLE_BITS ); i++) { +local_interrupt_disable (); +TEST_IRQ_OFF (); + } + + +for (int i =0 ; i <__IRQ_MASK (HARDIRQ_DISABLE_BITS ); i++) { +TEST_IRQ_OFF (); +local_interrupt_enable (); + } + +TEST_IRQ_ON (); + +local_interrupt_disable (); +TEST_IRQ_OFF (); +local_interrupt_enable (); +TEST_IRQ_ON (); +} +static void test_irq_save (struct kunit *test) {unsigned long flags;
@@ -79,6 +100,7 @@ static struct kunit_case test_cases[] = {KUNIT_CASE (test_single_irq_change),KUNIT_CASE (test_nested_irq_change),KUNIT_CASE (test_multiple_irq_change), +KUNIT_CASE (test_max_nesting_irq_change), KUNIT_CASE(test_irq_save), {}, };
Signature · 1 line
2.50.1 (Apple Git-155)
Re: [PATCH 0/4] Improvements on SpinLockIrq patch series
In reply to Boqun FengQuoted 17 lines · Boqun Feng
Hi Peter,
Per your request, I send those improvements as separate patches. I did a very light build and unit tests. I'm still waiting for the 0day build test report in case we missed something subtle, but figured maybe it's better if I send them out while I'm waiting.
Regards,
BoqunBoqun Feng (4):
irq: Add {over,under}flow detection forlocal_interrupt_{enable,disable}selftests/bpf: Use the new
NMI_BITSdefinition
irq: Explain better onNMI_MASKoverflow condition
irq: Add maxlocal_interrupt_disable()nesting level kunit test case
Quoted 6 lines
include/linux/hardirq.h | 2 +- include/linux/interrupt_rc.h | 13 +++++++++++ kernel/irq/refcount_interrupt_test.c | 22 +++++++++++++++++++ .../testing/selftests/bpf/bpf_experimental.h | 2 +- 4 files changed, 37 insertions(+), 2 deletions(-)
Looks good to me.