softirq: Consolidate and optimize softirq mask v3
Subsystems (5)
Thomas, Ingo,
Please pull the softirq/mask-v3 branch that can be found at:
git://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git
softirq/mask-v3
HEAD: e225bb16aa5eb0ab83ddd0de423aca3057b1cfee
(No change since last version, just a rebase against v4.17-rc1 and add acks from Michael Ellerman)
--
The softirq mask and its accessors/mutators have many implementations scattered around many architectures. Most do the same things consisting in a field in a per-cpu struct (often irq_cpustat_t) accessed through per-cpu ops. We can provide instead a generic efficient version that most of them can use. In fact s390 is the only exception because the field is stored in lowcore.
Thanks,
Frederic
---
Frederic Weisbecker (10):
ia64: Convert local_softirq_pending() to per-cpu ops
sparc: Convert local_softirq_pending() to use per-cpu op
softirq: Turn default irq_cpustat_t to standard per-cpu
softirq: Consolidate default local_softirq_pending() implementations
ia64: Switch to generic local_softirq_pending() implementation
parisc: Switch to generic local_softirq_pending() implementation
powerpc: Switch to generic local_softirq_pending() implementation
sparc: Switch to generic local_softirq_pending() implementation
x86: Switch to generic local_softirq_pending() implementation softirq/s390: Move default mutators of overwritten softirq mask to s390
arch/ia64/include/asm/hardirq.h | 2 +- arch/parisc/include/asm/hardirq.h | 8 -------- arch/powerpc/include/asm/hardirq.h | 7 ------- arch/s390/include/asm/hardirq.h | 2 ++ arch/sparc/include/asm/hardirq_64.h | 5 +++-- arch/x86/include/asm/hardirq.h | 8 -------- include/linux/interrupt.h | 13 ++++++++++--- include/linux/irq_cpustat.h | 10 +++------- kernel/softirq.c | 4 ++-- 9 files changed, 21 insertions(+), 38 deletions(-)
PATCH 1/10 ia64: Convert local_softirq_pending() to per-cpu ops
In order to consolidate and optimize generic softirq mask accesses, we first need to convert architectures to use per-cpu operations when possible.
--- arch/ia64/include/asm/hardirq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/ia64/include/asm/hardirq.h +1 -1
@@ -13,7 +13,7 @@#define __ARCH_IRQ_STAT 1 -#define local_softirq_pending () (local_cpu_data->softirq_pending) +#define local_softirq_pending () (*this_cpu_ptr(&ia64_cpu_info.softirq_pending))#include <linux/threads.h> #include <linux/irq.h>
Signature · 1 line
2.7.4
PATCH 2/10 sparc: Convert local_softirq_pending() to use per-cpu op
In reply to Frederic WeisbeckerIn order to consolidate and optimize generic softirq mask accesses, we first need to convert architectures to use per-cpu operations when possible.
--- arch/sparc/include/asm/hardirq_64.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/sparc/include/asm/hardirq_64.h +1 -1
@@ -11,7 +11,7 @@#define __ARCH_IRQ_STAT#define local_softirq_pending () \ - (local_cpu_data().__softirq_pending) + (*this_cpu_ptr(&__cpu_data.__softirq_pending))void ack_bad_irq (unsigned int irq);
Signature · 1 line
2.7.4
PATCH 3/10 softirq: Turn default irq_cpustat_t to standard per-cpu
In reply to Frederic WeisbeckerIn order to optimize and consolidate softirq mask accesses, let's convert the default irq_cpustat_t implementation to per-CPU standard API.
--- include/linux/irq_cpustat.h | 4 ++-- kernel/softirq.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-)
include/linux/irq_cpustat.h +2 -2
@@ -18,8 +18,8 @@*/ #ifndef __ARCH_IRQ_STAT -extern irq_cpustat_t irq_stat[];/* defined in asm/hardirq.h */ -#define __IRQ_STAT (cpu, member) (irq_stat[cpu].member) +DECLARE_PER_CPU_ALIGNED (irq_cpustat_t, irq_stat);/* defined in asm/hardirq.h */ +#define __IRQ_STAT (cpu, member) (per_cpu(irq_stat.member, cpu))#endif /* arch independent irq_stat fields */
kernel/softirq.c +2 -2
@@ -49,8 +49,8 @@*/ #ifndef __ARCH_IRQ_STAT -irq_cpustat_t irq_stat[NR_CPUS ]____cacheline_aligned ; -EXPORT_SYMBOL (irq_stat); +DEFINE_PER_CPU_ALIGNED (irq_cpustat_t, irq_stat); +EXPORT_PER_CPU_SYMBOL (irq_stat);#endif static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
Signature · 1 line
2.7.4
PATCH 4/10 softirq: Consolidate default local_softirq_pending() implementations
In reply to Frederic WeisbeckerConsolidate and optimize default softirq mask API implementations. Per-CPU operations are expected to be faster and a few architectures already rely on them to implement local_softirq_pending() and related accessors/mutators. Those will be migrated to the new generic code.
--- include/linux/interrupt.h | 14 ++++++++++++++ include/linux/irq_cpustat.h | 6 +----- 2 files changed, 15 insertions(+), 5 deletions(-)
include/linux/interrupt.h +14 -0
@@ -432,11 +432,25 @@ extern bool force_irqthreads;#define force_irqthreads (0)#endif +#ifndef local_softirq_pending + +#ifndef local_softirq_pending_ref +#define local_softirq_pending_ref irq_stat.__softirq_pending +#endif + +#define local_softirq_pending () (__this_cpu_read(local_softirq_pending_ref)) +#define set_softirq_pending (x) (__this_cpu_write(local_softirq_pending_ref, (x))) +#define or_softirq_pending (x) (__this_cpu_or(local_softirq_pending_ref, (x))) + +#else /* local_softirq_pending */ +#ifndef __ARCH_SET_SOFTIRQ_PENDING#define set_softirq_pending (x) (local_softirq_pending() = (x))#define or_softirq_pending (x) (local_softirq_pending() |= (x))#endif +#endif /* local_softirq_pending */ +/* Some architectures might implement lazy enabling/disabling of * interrupts. In some cases, such as stop_machine, we might want * to ensure that after a local_irq_disable(), interrupts have
include/linux/irq_cpustat.h +1 -5
@@ -22,11 +22,7 @@ DECLARE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat); /* defined in asm/hardirq.h */#define __IRQ_STAT (cpu, member) (per_cpu(irq_stat.member, cpu))#endif -/* arch independent irq_stat fields */ -#define local_softirq_pending () \ - __IRQ_STAT(smp_processor_id(), __softirq_pending) - -/* arch dependent irq_stat fields */ +/* arch dependent irq_stat fields */ #define nmi_count (cpu) __IRQ_STAT((cpu), __nmi_count)/* i386 */ #endif /* __irq_cpustat_h */
Signature · 1 line
2.7.4
PATCH 5/10 ia64: Switch to generic local_softirq_pending() implementation
In reply to Frederic WeisbeckerBenefit from the generic softirq mask implementation that rely on per-CPU mutators instead of working with raw operators on top of this_cpu_ptr().
--- arch/ia64/include/asm/hardirq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/ia64/include/asm/hardirq.h +1 -1
@@ -13,7 +13,7 @@#define __ARCH_IRQ_STAT 1 -#define local_softirq_pending () (*this_cpu_ptr(&ia64_cpu_info.softirq_pending)) +#define local_softirq_pending_ref ia64_cpu_info.softirq_pending#include <linux/threads.h> #include <linux/irq.h>
Signature · 1 line
2.7.4
PATCH 6/10 parisc: Switch to generic local_softirq_pending() implementation
In reply to Frederic WeisbeckerRemove the ad-hoc implementation, the generic code now allows us not to reinvent the wheel.
--- arch/parisc/include/asm/hardirq.h | 8 -------- 1 file changed, 8 deletions(-)
arch/parisc/include/asm/hardirq.h +0 -8
@@ -34,14 +34,6 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define __IRQ_STAT (cpu, member) (irq_stat[cpu].member)#define inc_irq_stat (member) this_cpu_inc(irq_stat.member)#define __inc_irq_stat (member) __this_cpu_inc(irq_stat.member) -#define local_softirq_pending () this_cpu_read(irq_stat.__softirq_pending) - -#define __ARCH_SET_SOFTIRQ_PENDING - -#define set_softirq_pending (x) \ - this_cpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) this_cpu_or(irq_stat.__softirq_pending, (x)) -#define ack_bad_irq (irq) WARN(1, "unexpected IRQ trap at vector %02x\n", irq)#endif /* _PARISC_HARDIRQ_H */
Signature · 1 line
2.7.4
PATCH 7/10 powerpc: Switch to generic local_softirq_pending() implementation
In reply to Frederic WeisbeckerRemove the ad-hoc implementation, the generic code now allows us not to reinvent the wheel.
--- arch/powerpc/include/asm/hardirq.h | 7 ------- 1 file changed, 7 deletions(-)
arch/powerpc/include/asm/hardirq.h +0 -7
@@ -25,15 +25,8 @@ typedef struct {DECLARE_PER_CPU_SHARED_ALIGNED (irq_cpustat_t, irq_stat);#define __ARCH_IRQ_STAT - -#define local_softirq_pending () __this_cpu_read(irq_stat.__softirq_pending) - -#define __ARCH_SET_SOFTIRQ_PENDING#define __ARCH_IRQ_EXIT_IRQS_DISABLED -#define set_softirq_pending (x) __this_cpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) __this_cpu_or(irq_stat.__softirq_pending, (x)) -static inline void ack_bad_irq (unsigned int irq) {printk (KERN_CRIT "unexpected IRQ trap at vector %02x\n" , irq);
Signature · 1 line
2.7.4
PATCH 8/10 sparc: Switch to generic local_softirq_pending() implementation
In reply to Frederic WeisbeckerBenefit from the generic softirq mask implementation that rely on per-CPU mutators instead of working with raw operators on top of this_cpu_ptr().
--- arch/sparc/include/asm/hardirq_64.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
arch/sparc/include/asm/hardirq_64.h +3 -2
@@ -10,8 +10,9 @@#include <asm/cpudata.h> #define __ARCH_IRQ_STAT -#define local_softirq_pending () \ - (*this_cpu_ptr(&__cpu_data.__softirq_pending)) + +#define local_softirq_pending_ref \ + __cpu_data.__softirq_pendingvoid ack_bad_irq (unsigned int irq);
Signature · 1 line
2.7.4
PATCH 9/10 x86: Switch to generic local_softirq_pending() implementation
In reply to Frederic WeisbeckerRemove the ad-hoc implementation, the generic code now allows us not to reinvent the wheel.
--- arch/x86/include/asm/hardirq.h | 8 -------- 1 file changed, 8 deletions(-)
arch/x86/include/asm/hardirq.h +0 -8
@@ -50,14 +50,6 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);#define inc_irq_stat (member) this_cpu_inc(irq_stat.member) -#define local_softirq_pending () this_cpu_read(irq_stat.__softirq_pending) - -#define __ARCH_SET_SOFTIRQ_PENDING - -#define set_softirq_pending (x) \ - this_cpu_write(irq_stat.__softirq_pending, (x)) -#define or_softirq_pending (x) this_cpu_or(irq_stat.__softirq_pending, (x)) -extern void ack_bad_irq (unsigned int irq);extern u64 arch_irq_stat_cpu (unsigned int cpu);
Signature · 1 line
2.7.4
PATCH 10/10 softirq/s390: Move default mutators of overwritten softirq mask to s390
In reply to Frederic Weisbeckers390 is now the last architecture that entirely overwrites local_softirq_pending() and uses the according default definitions of set_softirq_pending() and or_softirq_pending().
Just move these to s390 to debloat the generic code complexity.
--- arch/s390/include/asm/hardirq.h | 2 ++ include/linux/interrupt.h | 7 ------- 2 files changed, 2 insertions(+), 7 deletions(-)
arch/s390/include/asm/hardirq.h +2 -0
@@ -14,6 +14,8 @@#include <asm/lowcore.h> #define local_softirq_pending () (S390_lowcore.softirq_pending) +#define set_softirq_pending (x) (S390_lowcore.softirq_pending = (x)) +#define or_softirq_pending (x) (S390_lowcore.softirq_pending |= (x))#define __ARCH_IRQ_STAT#define __ARCH_HAS_DO_SOFTIRQ
include/linux/interrupt.h +0 -7
@@ -442,13 +442,6 @@ extern bool force_irqthreads;#define set_softirq_pending (x) (__this_cpu_write(local_softirq_pending_ref, (x)))#define or_softirq_pending (x) (__this_cpu_or(local_softirq_pending_ref, (x))) -#else /* local_softirq_pending */ - -#ifndef __ARCH_SET_SOFTIRQ_PENDING -#define set_softirq_pending (x) (local_softirq_pending() = (x)) -#define or_softirq_pending (x) (local_softirq_pending() |= (x)) -#endif -#endif /* local_softirq_pending */ /* Some architectures might implement lazy enabling/disabling of
Signature · 1 line
2.7.4