Make Generic irq_stat Structure Use per-cpu
[ Sorry about the counting thing, decided to split this patch after I'd
sent the first two. ]
Changes the generic irq_stat structure to use DECLARE_PER_CPU. This saves a little space, and is more efficient on some archs.
Only i386 is fixed here.
Some archs refer to this in assembler: this is easy on UP (append __per_cpu to the symbol name), but more complex on SMP (you need to add __per_cpu_offset[smp_processor_id()]). If that's too hard then you can just stop using the generic structure. Please remember to drop the ____cacheline_aligned from the typedef of irq_cpustat_t!
Name: Make Generic irq_stat Structure Use per-cpu
Author: Rusty Russell
Status: Tested on 2.5.74
Depends: Percpu/local_softirq_pending.patch.gz
D: Changes the generic irq_stat structure to use DECLARE_PER_CPU.
D: This saves a little space, and is more efficient on some archs. D:
D: Only i386 is fixed here.
D:
D: Some archs refer to this in assembler: this is easy on UP (append
D: __per_cpu to the symbol name), but more complex on SMP (you need
D: to add __per_cpu_offset[smp_processor_id()]). If that's too hard
D: then you can just stop using the generic structure.
D: Please remember to drop the ____cacheline_aligned from the typedef
D: of irq_cpustat_t!
arch/i386/kernel/apic.c +1 -3
@@ -1039,12 +1039,10 @@ inline void smp_local_timer_interrupt(stvoid smp_apic_timer_interrupt (struct pt_regs regs) { -int cpu =smp_processor_id (); -/* * the NMI deadlock-detector uses this. */ - irq_stat[cpu].apic_timer_irqs++; +__get_cpu_var (irq_stat).apic_timer_irqs++;/* * NOTE! We'd better ACK the irq immediately,
arch/i386/kernel/io_apic.c +1 -1
@@ -304,7 +304,7 @@ struct irq_cpu_info {#define IRQ_DELTA (cpu,irq) (irq_cpu_data[cpu].irq_delta[irq])#define IDLE_ENOUGH (cpu,now) \ - (idle_cpu(cpu) && ((now) - irq_stat[(cpu)].idle_timestamp > 1)) + (idle_cpu(cpu) && ((now) - per_cpu(irq_stat,(cpu)).idle_timestamp > 1))#define IRQ_ALLOWED (cpu,allowed_mask) \ ((1 << cpu) & (allowed_mask))
arch/i386/kernel/irq.c +2 -1
@@ -181,7 +181,8 @@ skip:seq_printf (p,"LOC: " );for (j =0 ; j <NR_CPUS ; j++)if (cpu_online (j)) -seq_printf (p,"%10u " , irq_stat[j].apic_timer_irqs); +seq_printf (p,"%10u " , +per_cpu (irq_stat, j).apic_timer_irqs);seq_putc (p,'\n' );#endif seq_printf (p,"ERR: %10u\n" ,atomic_read (&irq_err_count));
arch/i386/kernel/nmi.c +2 -2
@@ -90,7 +90,7 @@ int __init check_nmi_watchdog (void)printk (KERN_INFO "testing NMI watchdog ... " );for (cpu =0 ; cpu <NR_CPUS ; cpu++) - prev_nmi_count[cpu] = irq_stat[cpu].__nmi_count; + prev_nmi_count[cpu] =nmi_count (cpu);local_irq_enable ();mdelay ((10 *1000 )/nmi_hz);// wait 10 ticks
@@ -409,7 +409,7 @@ void nmi_watchdog_tick (struct pt_regs **/ int sum, cpu =smp_processor_id (); - sum = irq_stat[cpu].apic_timer_irqs; + sum =__get_cpu_var (irq_stat).apic_timer_irqs;if (last_irq_sums[cpu] == sum) {/*
arch/i386/kernel/process.c +1 -1
@@ -141,7 +141,7 @@ void cpu_idle (void)void (*idle)(void ) = pm_idle;if (!idle) idle = default_idle; - irq_stat[smp_processor_id ()].idle_timestamp = jiffies; +__get_cpu_var (irq_stat).idle_timestamp = jiffies;while (!need_resched ())idle ();schedule ();
arch/i386/kernel/traps.c +1 -1
@@ -475,7 +475,7 @@ asmlinkage void do_nmi(struct pt_regs *nmi_enter (); cpu =smp_processor_id (); - ++nmi_count (cpu); + ++local_nmi_count ();if (!nmi_callback (regs, cpu))default_do_nmi (regs);
arch/x86_64/kernel/irq.c +1 -1
@@ -173,7 +173,7 @@ skip:seq_printf (p,"NMI: " );for (j =0 ; j <NR_CPUS ; j++)if (cpu_online (j)) -seq_printf (p,"%10u " , cpu_pda[j].__nmi_count); +seq_printf (p,"%10u " ,nmi_count (j));seq_putc (p,'\n' );#ifdef CONFIG_X86_LOCAL_APIC seq_printf (p,"LOC: " );
arch/x86_64/kernel/nmi.c +3 -3
@@ -86,17 +86,17 @@ int __init check_nmi_watchdog (void)printk (KERN_INFO "testing NMI watchdog ... " );for (cpu =0 ; cpu <NR_CPUS ; cpu++) - counts[cpu] = cpu_pda[cpu].__nmi_count; + counts[cpu] =nmi_count (cpu);local_irq_enable ();mdelay ((10 *1000 )/nmi_hz);// wait 10 ticks for (cpu =0 ; cpu <NR_CPUS ; cpu++) {if (!cpu_online (cpu))continue ; -if (cpu_pda[cpu].__nmi_count - counts[cpu] <=5 ) { +if (nmi_count (cpu) - counts[cpu] <=5 ) {printk ("CPU#%d: NMI appears to be stuck (%d)!\n" , cpu, - cpu_pda[cpu].__nmi_count); +nmi_count (cpu)); nmi_active =0 ;return -1 ; }
include/asm-i386/hardirq.h +1 -1
@@ -10,7 +10,7 @@ typedef struct {unsigned long idle_timestamp;unsigned int __nmi_count;/* arch dependent */ unsigned int apic_timer_irqs;/* arch dependent */ -}____cacheline_aligned irq_cpustat_t; +} irq_cpustat_t;#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
include/linux/irq_cpustat.h +8 -9
@@ -10,6 +10,7 @@*/ #include <linux/config.h> +#include <linux/percpu.h> /* * Simple wrappers reducing source bloat. Define all irq_stat fields
@@ -18,19 +19,17 @@*/ #ifndef __ARCH_IRQ_STAT -extern irq_cpustat_t irq_stat[];/* defined in asm/hardirq.h */ -#ifdef CONFIG_SMP -#define __IRQ_STAT (cpu, member) (irq_stat[cpu].member) -#else -#define __IRQ_STAT (cpu, member) ((void)(cpu), irq_stat[0].member) -#endif +DECLARE_PER_CPU (irq_cpustat_t, irq_stat);/* defined in asm/hardirq.h */ +#define __IRQ_STAT (cpu, member) (per_cpu(irq_stat, (cpu)).member) +#define __LOCAL_IRQ_STAT (member) (__get_cpu_var(irq_stat).member)#endif /* arch independent irq_stat fields */ -#define softirq_pending (cpu) __IRQ_STAT((cpu), __softirq_pending) -#define local_softirq_pending () softirq_pending(smp_processor_id()) +#define local_softirq_pending () __LOCAL_IRQ_STAT(__softirq_pending)/* arch dependent irq_stat fields */ -#define nmi_count (cpu) __IRQ_STAT((cpu), __nmi_count)/* i386 */ +/* i386, x86-64 */ +#define nmi_count (cpu) __IRQ_STAT((cpu), __nmi_count) +#define local_nmi_count () __LOCAL_IRQ_STAT(__nmi_count)#endif /* __irq_cpustat_h */
kernel/softirq.c +2 -2
@@ -36,8 +36,8 @@*/ #ifndef __ARCH_IRQ_STAT -irq_cpustat_t irq_stat[NR_CPUS ]____cacheline_aligned ; -EXPORT_SYMBOL (irq_stat); +DEFINE_PER_CPU (irq_cpustat_t, irq_stat); +EXPORT_PER_CPU_SYMBOL (irq_stat);#endif static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
--
Anyone who quotes me in their sig is an idiot. -- Rusty Russell.