2.5.17 apic_timer_irqs cacheline bouncing
PATCH 1/1 2.5.17 apic_timer_irqs cacheline bouncing
The following patch extirpates cacheline bouncing due to apic_timer_irqs[NR_CPUS] array, by placing them in the irq_stat[] cacheline aligned array for i386, and cpu_pda[] array for x86_64. (irq_stat structure is moved to cpu_pda in x86_64). apic_timer_irqs get updated at every local apic timer interrput, causing the cacheline to bounce. I could have used Rusty's per-cpu area mechanism, but I wouldn't be able to make a patch for 2.4, and irq_stat seemed like a good place for apic_timer_irqs.
This patch has been tested on a 4 way PIII xeon. I hope I have got the x86_64 stuff right. The ia64 part is probably dead code lying around to make irq.c platform independent in the future. Please consider for 2.5.18*
Kiran
arch/i386/kernel/apic.c +1 -2
@@ -1063,7 +1063,6 @@* [ if a single-CPU system runs an SMP kernel then we call the local * interrupt as well. Thus we cannot inline the local irq ... ] */ -unsigned int apic_timer_irqs [NR_CPUS ];void smp_apic_timer_interrupt (struct pt_regs regs) {
@@ -1072,7 +1071,7 @@/* * the NMI deadlock-detector uses this. */ - apic_timer_irqs[cpu]++; + irq_stat[cpu].apic_timer_irqs++;/* * NOTE! We'd better ACK the irq immediately,
arch/i386/kernel/irq.c +2 -1
@@ -168,7 +168,8 @@#if CONFIG_X86_LOCAL_APIC seq_printf (p,"LOC: " );for (j =0 ; j < smp_num_cpus; j++) -seq_printf (p,"%10u " , apic_timer_irqs[cpu_logical_map (j)]); +seq_printf (p,"%10u " , + irq_stat[cpu_logical_map (j)].apic_timer_irqs);seq_putc (p,'\n' );#endif seq_printf (p,"ERR: %10u\n" ,atomic_read (&irq_err_count));
arch/i386/kernel/nmi.c +1 -1
@@ -344,7 +344,7 @@*/ int sum, cpu =smp_processor_id (); - sum = apic_timer_irqs[cpu]; + sum = irq_stat[cpu].apic_timer_irqs;if (last_irq_sums[cpu] == sum) {/*
arch/ia64/kernel/irq.c +1 -1
@@ -189,7 +189,7 @@seq_puts (p,"LOC: " );for (j =0 ; j < smp_num_cpus; j++)seq_printf (p,"%10u " , - apic_timer_irqs[cpu_logical_map (j)]); + irq_stat[cpu_logical_map (j)].apic_timer_irqs);seq_putc (p,'\n' );#endif seq_printf (p,"ERR: %10u\n" ,atomic_read (&irq_err_count));
arch/x86_64/kernel/apic.c +1 -2
@@ -1044,7 +1044,6 @@* [ if a single-CPU system runs an SMP kernel then we call the local * interrupt as well. Thus we cannot inline the local irq ... ] */ -unsigned int apic_timer_irqs [NR_CPUS ];void smp_apic_timer_interrupt (struct pt_regs *regs) {
@@ -1053,7 +1052,7 @@/* * the NMI deadlock-detector uses this. */ - apic_timer_irqs[cpu]++; + cpu_pda[cpu].apic_timer_irqs++;/* * NOTE! We'd better ACK the irq immediately,
arch/x86_64/kernel/irq.c +1 -1
@@ -168,7 +168,7 @@#if CONFIG_X86_LOCAL_APIC seq_printf (p,"LOC: " );for (j =0 ; j < smp_num_cpus; j++) -seq_printf (p,"%10u " , apic_timer_irqs[cpu_logical_map (j)]); +seq_printf (p,"%10u " , cpu_pda[cpu_logical_map (j)].apic_timer_irqs);seq_putc (p,'\n' );#endif seq_printf (p,"ERR: %10u\n" ,atomic_read (&irq_err_count));
arch/x86_64/kernel/nmi.c +1 -1
@@ -240,7 +240,7 @@*/ int sum, cpu =smp_processor_id (); - sum = apic_timer_irqs[cpu]; + sum = cpu_pda[cpu].apic_timer_irqs;if (last_irq_sums[cpu] == sum) {/*
include/asm-i386/apic.h +0 -1
@@ -86,7 +86,6 @@extern struct pm_dev *apic_pm_register (pm_dev_t ,unsigned long ,pm_callback );extern void apic_pm_unregister (struct pm_dev *); -extern unsigned int apic_timer_irqs [NR_CPUS ];extern int check_nmi_watchdog (void );extern unsigned int nmi_watchdog;
include/asm-i386/hardirq.h +1 -0
@@ -14,6 +14,7 @@struct task_struct * __ksoftirqd_task;/* waitqueue is too large */ unsigned long idle_timestamp;unsigned int __nmi_count;/* arch dependent */ +unsigned int apic_timer_irqs;/* arch dependent */ }____cacheline_aligned irq_cpustat_t;#include <linux/irq_cpustat.h> /* Standard mappings for irq_cpustat_t above */
include/asm-x86_64/apic.h +0 -1
@@ -86,7 +86,6 @@extern struct pm_dev *apic_pm_register (pm_dev_t ,unsigned long ,pm_callback );extern void apic_pm_unregister (struct pm_dev *); -extern unsigned int apic_timer_irqs [NR_CPUS ];extern int check_nmi_watchdog (void );extern unsigned int nmi_watchdog;
include/asm-x86_64/pda.h +1 -0
@@ -23,6 +23,7 @@unsigned int __local_irq_count;unsigned int __local_bh_count;unsigned int __nmi_count;/* arch dependent */ +unsigned int apic_timer_irqs;/* arch dependent */ struct task_struct * __ksoftirqd_task;/* waitqueue is too large */ }____cacheline_aligned ;
kernel/softirq.c +1 -1
@@ -40,7 +40,7 @@- Bottom halves: globally serialized, grr... */ -irq_cpustat_t irq_stat[NR_CPUS ]; +irq_cpustat_t irq_stat[NR_CPUS ]____cacheline_aligned ; static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;