Skip to contentSkip to conversation

Disable SW irqbalance/irqaffinity for E7520/E7320/E7525

PATCH In review Suresh Siddha 5 participants 11 messages Sep 24, 2004 – Oct 1, 2004 PCI

lore
Share & tools
lore

Suresh Siddha

author
#0 lore raw

cover Disable SW irqbalance/irqaffinity for E7520/E7320/E7525

As part of the workaround for the "Interrupt message re-ordering across hub interface" errata (page #16 in
http://developer.intel.com/design/chipsets/specupdt/30288402.pdf),
BIOS may enable hardware IRQ balancing for Lindenhurst/Tumwater based chipset platforms. Software based irq_balance/irq_affinity should be disabled if hardware IRQ balancing is enabled.

This is applicable for chipsets with PCI id's

E7520, 0x3590
E7320, 0x3592
E7525, 0x359E

and with revision ID 0x09 and below.

Patch is broken into two parts.

1/2 - Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline with x86 mach-default
2/2 - Add pci quirks to disable irq_balance/affinity based on the above information and make sure destination cpus in IO-APIC redirection table entries are set to cpu_online_map

Signed-off-by Suresh Siddha <[email protected]>
#1 lore raw
Quoted 6 lines · Suresh Siddha

As part of the workaround for the "Interrupt message re-ordering across hub interface" errata (page #16 in
http://developer.intel.com/design/chipsets/specupdt/30288402.pdf),
BIOS may enable hardware IRQ balancing for Lindenhurst/Tumwater based chipset platforms. Software based irq_balance/irq_affinity should be disabled if hardware IRQ balancing is enabled.

Can we avoid those tests? Like not starting the irq balancer code at all with those chipsets?

Thanks,
Zwane

#2 lore raw
Quoted 11 lines · Zwane Mwaikambo
Quoted 6 lines · Suresh Siddha

As part of the workaround for the "Interrupt message re-ordering across hub interface" errata (page #16 in
http://developer.intel.com/design/chipsets/specupdt/30288402.pdf),
BIOS may enable hardware IRQ balancing for Lindenhurst/Tumwater based chipset platforms. Software based irq_balance/irq_affinity should be disabled if hardware IRQ balancing is enabled.

Quoted 1 line

Can we avoid those tests? Like not starting the irq balancer code at all with those chipsets?

That's my preference.

Or, compile with !CONFIG_KIRQD -- the recommended setting for modern distros, since irqbalanced(1) is shipped.

Jeff

author
#3 lore raw
Quoted 12 lines · Jeff Garzik
Quoted 11 lines · Zwane Mwaikambo
Quoted 6 lines · Suresh Siddha

As part of the workaround for the "Interrupt message re-ordering across hub interface" errata (page #16 in
http://developer.intel.com/design/chipsets/specupdt/30288402.pdf),
BIOS may enable hardware IRQ balancing for Lindenhurst/Tumwater based chipset platforms. Software based irq_balance/irq_affinity should be disabled if hardware IRQ balancing is enabled.

Quoted 6 lines
Quoted 2 lines

Can we avoid those tests? Like not starting the irq balancer code at all with those chipsets?

That's my preference.

Ok. How about this patch?

Add pci quirks to disable irqbalance/affinity for E7520/E7320/E7525 with revision ID 0x09 and below.

Signed-off-by Suresh Siddha <[email protected]>
arch/i386/kernel/io_apic.c +29 -2
diff -Nru linux-2.6.9-rc2/arch/i386/kernel/io_apic.c linux-irq/arch/i386/kernel/io_apic.c
@@ -31,6 +31,7 @@
 #include <linux/mc146818rtc.h>
 #include <linux/compiler.h>
 #include <linux/acpi.h>
+#include <linux/pci.h>
 
 #include <linux/sysdev.h>
 #include <asm/io.h>

@@ -260,7 +261,7 @@
 cpumask_t __cacheline_aligned pending_irq_balance_cpumask[NR_IRQS];
 
 #define IRQBALANCE_CHECK_ARCH -999
-static int irqbalance_disabled = IRQBALANCE_CHECK_ARCH;
+int irqbalance_disabled = IRQBALANCE_CHECK_ARCH;
 static int physical_balance = 0;
 
 struct irq_cpu_info {

@@ -582,6 +583,30 @@
 	return 0;
 }
 
+/* HW irqbalance may be enabled for E7520/E7320/E7525 with revision ID 
+ * 0x09 and below. Disable sw irqbalance in that case
+ */
+static void check_hw_irqbalance(void)
+{
+	struct pci_dev *dev = NULL;
+	extern void quirk_intel_irqbalance(struct pci_dev *dev);
+
+	dev = pci_find_device(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_SMCH, 
+			      dev);
+	if (!dev)
+		dev = pci_find_device(PCI_VENDOR_ID_INTEL, 
+				      PCI_DEVICE_ID_INTEL_E7320_MCH, dev);
+	if (!dev)
+		dev = pci_find_device(PCI_VENDOR_ID_INTEL, 
+				      PCI_DEVICE_ID_INTEL_E7525_MCH, dev);
+	if (!dev)
+		return;
+
+	quirk_intel_irqbalance(dev);
+
+	return;	
+}
+
 static int __init balanced_irq_init(void)
 {
 	int i;

@@ -590,6 +615,8 @@
 
 	cpus_shift_right(tmp, cpu_online_map, 2);
         c = &boot_cpu_data;
+
+	check_hw_irqbalance();
 	/* When not overwritten by the command line ask subarchitecture. */
 	if (irqbalance_disabled == IRQBALANCE_CHECK_ARCH)
 		irqbalance_disabled = NO_BALANCE_IRQ;

@@ -636,7 +663,7 @@
 	return 0;
 }
 
-static int __init irqbalance_disable(char *str)
+int __init irqbalance_disable(char *str)
 {
 	irqbalance_disabled = 1;
 	return 0;
arch/i386/kernel/irq.c +4 -2
diff -Nru linux-2.6.9-rc2/arch/i386/kernel/irq.c linux-irq/arch/i386/kernel/irq.c
@@ -272,7 +272,7 @@
 
 static int noirqdebug;
 
-static int __init noirqdebug_setup(char *str)
+int __init noirqdebug_setup(char *str)
 {
 	noirqdebug = 1;
 	printk("IRQ lockup detection disabled\n");

@@ -997,13 +997,15 @@
 	return len;
 }
 
+int no_irq_affinity;
+
 static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
 					unsigned long count, void *data)
 {
 	int irq = (long)data, full_count = count, err;
 	cpumask_t new_value, tmp;
 
-	if (!irq_desc[irq].handler->set_affinity)
+	if (!irq_desc[irq].handler->set_affinity || no_irq_affinity)
 		return -EIO;
 
 	err = cpumask_parse(buffer, count, new_value);
arch/x86_64/kernel/irq.c +3 -1
diff -Nru linux-2.6.9-rc2/arch/x86_64/kernel/irq.c linux-irq/arch/x86_64/kernel/irq.c
@@ -835,6 +835,8 @@
 	return len;
 }
 
+int no_irq_affinity;
+
 static int irq_affinity_write_proc (struct file *file,
 					const char __user *buffer,
 					unsigned long count, void *data)

@@ -842,7 +844,7 @@
 	int irq = (long) data, full_count = count, err;
 	cpumask_t tmp, new_value;
 
-	if (!irq_desc[irq].handler->set_affinity)
+	if (!irq_desc[irq].handler->set_affinity || no_irq_affinity)
 		return -EIO;
 
 	err = cpumask_parse(buffer, count, new_value);
drivers/pci/quirks.c +58 -0

64 lines not on this page · raw message

include/linux/pci_ids.h +2 -0
diff -Nru linux-2.6.9-rc2/include/linux/pci_ids.h linux-irq/include/linux/pci_ids.h
@@ -2204,6 +2204,8 @@
 #define PCI_DEVICE_ID_INTEL_82855GM_HB	0x3580
 #define PCI_DEVICE_ID_INTEL_82855GM_IG	0x3582
 #define PCI_DEVICE_ID_INTEL_SMCH	0x3590
+#define PCI_DEVICE_ID_INTEL_E7320_MCH	0x3592
+#define PCI_DEVICE_ID_INTEL_E7525_MCH	0x359e
 #define PCI_DEVICE_ID_INTEL_80310	0x530d
 #define PCI_DEVICE_ID_INTEL_82371SB_0	0x7000
 #define PCI_DEVICE_ID_INTEL_82371SB_1	0x7010
#4 lore raw

Hello Suresh,

Quoted 9 lines · Suresh Siddha

Ok. How about this patch?

Add pci quirks to disable irqbalance/affinity for E7520/E7320/E7525 with revision ID 0x09 and below.

diff -Nru linux-2.6.9-rc2/drivers/pci/quirks.c linux-irq/drivers/pci/quirks.c

drivers/pci/quirks.c line 823

Quoted 10 lines
drivers/pci/quirks.c +7 -0
@@ -814,6 +814,64 @@
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82801DB_12,	asus_hides_smbus_lpc );
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82801EB_0,	asus_hides_smbus_lpc );
 
+#ifdef CONFIG_X86_IO_APIC
+#include <asm/hw_irq.h>
+#ifdef CONFIG_IRQBALANCE
+extern int irqbalance_disable(char *str);
+#endif
+extern int no_irq_affinity;
+extern int noirqdebug_setup(char *str);

Ok this is sort of ugly, but it's not your fault, i understand that the PCI quirks code is too late after IOAPIC setup, x86_64 has some early PCI bridge detection code which helps in doing IOAPIC quirks.

drivers/pci/quirks.c line 826

Quoted 1 line
drivers/pci/quirks.c +1 -0
+void __devinit quirk_intel_irqbalance(struct pci_dev *dev)

This may as well be moved elsewhere since it's not actually going to be used in PCI quirks. I think you should just do the chipset detection in io_apic.c and then do the disable from there, it's racy and strange (although it may seem natural) to do it in quirks.c

Thanks,
Zwane

author
#5 lore raw

Zwane,

As far as irq_affinity is concerned, workaround fits nicely in quirks.c infrastructure.

x86 irqbalance is the one which is causing the pain. Workaround will be more cleaner if we can move balanced_irq_init() to late_initcall.

If there is no objection, I can post a new patch with that change.

thanks,
suresh

Quoted 19 lines · Zwane Mwaikambo

drivers/pci/quirks.c line 823

Quoted 7 lines · Suresh Siddha
drivers/pci/quirks.c +7 -0
+#ifdef CONFIG_X86_IO_APIC
+#include <asm/hw_irq.h>
+#ifdef CONFIG_IRQBALANCE
+extern int irqbalance_disable(char *str);
+#endif
+extern int no_irq_affinity;
+extern int noirqdebug_setup(char *str);

Ok this is sort of ugly, but it's not your fault, i understand that the PCI quirks code is too late after IOAPIC setup, x86_64 has some early PCI bridge detection code which helps in doing IOAPIC quirks.

Quoted 1 line

+void __devinit quirk_intel_irqbalance(struct pci_dev *dev)

This may as well be moved elsewhere since it's not actually going to be used in PCI quirks. I think you should just do the chipset detection in io_apic.c and then do the disable from there, it's racy and strange (although it may seem natural) to do it in quirks.c

author
#6 lore raw
Quoted 7 lines · Suresh Siddha

As far as irq_affinity is concerned, workaround fits nicely in quirks.c infrastructure.

x86 irqbalance is the one which is causing the pain. Workaround will be more cleaner if we can move balanced_irq_init() to late_initcall.

If there is no objection, I can post a new patch with that change.

Here is a new patch. Andrew please apply.

thanks,
suresh

--

As part of the workaround for the "Interrupt message re-ordering across hub interface" errata (page #16 in
http://developer.intel.com/design/chipsets/specupdt/30288402.pdf),
BIOS may enable hardware IRQ balancing for E7520/E7320/E7525(revision ID 0x9 and below) based platforms.

Add pci quirks to disable SW irqbalance/affinity on those platforms. Move balanced_irq_init() to late_initcall so that kirqd will be started after pci quirks.

Signed-off-by Suresh Siddha <[email protected]>
arch/i386/kernel/io_apic.c +2 -2
diff -Nru linux-2.6.9-rc2/arch/i386/kernel/io_apic.c linux-irq/arch/i386/kernel/io_apic.c
@@ -636,7 +636,7 @@
 	return 0;
 }
 
-static int __init irqbalance_disable(char *str)
+int __init irqbalance_disable(char *str)
 {
 	irqbalance_disabled = 1;
 	return 0;

@@ -653,7 +653,7 @@
 	}
 }
 
-__initcall(balanced_irq_init);
+late_initcall(balanced_irq_init);
 
 #else /* !CONFIG_IRQBALANCE */
 static inline void move_irq(int irq) { }
arch/i386/kernel/irq.c +4 -2
diff -Nru linux-2.6.9-rc2/arch/i386/kernel/irq.c linux-irq/arch/i386/kernel/irq.c
@@ -272,7 +272,7 @@
 
 static int noirqdebug;
 
-static int __init noirqdebug_setup(char *str)
+int __init noirqdebug_setup(char *str)
 {
 	noirqdebug = 1;
 	printk("IRQ lockup detection disabled\n");

@@ -997,13 +997,15 @@
 	return len;
 }
 
+int no_irq_affinity;
+
 static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
 					unsigned long count, void *data)
 {
 	int irq = (long)data, full_count = count, err;
 	cpumask_t new_value, tmp;
 
-	if (!irq_desc[irq].handler->set_affinity)
+	if (!irq_desc[irq].handler->set_affinity || no_irq_affinity)
 		return -EIO;
 
 	err = cpumask_parse(buffer, count, new_value);
arch/x86_64/kernel/irq.c +3 -1
diff -Nru linux-2.6.9-rc2/arch/x86_64/kernel/irq.c linux-irq/arch/x86_64/kernel/irq.c
@@ -835,6 +835,8 @@
 	return len;
 }
 
+int no_irq_affinity;
+
 static int irq_affinity_write_proc (struct file *file,
 					const char __user *buffer,
 					unsigned long count, void *data)

@@ -842,7 +844,7 @@
 	int irq = (long) data, full_count = count, err;
 	cpumask_t tmp, new_value;
 
-	if (!irq_desc[irq].handler->set_affinity)
+	if (!irq_desc[irq].handler->set_affinity || no_irq_affinity)
 		return -EIO;
 
 	err = cpumask_parse(buffer, count, new_value);
drivers/pci/quirks.c +49 -0

55 lines not on this page · raw message

include/linux/pci_ids.h +2 -0
diff -Nru linux-2.6.9-rc2/include/linux/pci_ids.h linux-irq/include/linux/pci_ids.h
@@ -2204,6 +2204,8 @@
 #define PCI_DEVICE_ID_INTEL_82855GM_HB	0x3580
 #define PCI_DEVICE_ID_INTEL_82855GM_IG	0x3582
 #define PCI_DEVICE_ID_INTEL_SMCH	0x3590
+#define PCI_DEVICE_ID_INTEL_E7320_MCH	0x3592
+#define PCI_DEVICE_ID_INTEL_E7525_MCH	0x359e
 #define PCI_DEVICE_ID_INTEL_80310	0x530d
 #define PCI_DEVICE_ID_INTEL_82371SB_0	0x7000
 #define PCI_DEVICE_ID_INTEL_82371SB_1	0x7010
#7 lore raw
Quoted 10 lines · Suresh Siddha
Quoted 7 lines · Suresh Siddha

As far as irq_affinity is concerned, workaround fits nicely in quirks.c infrastructure.

x86 irqbalance is the one which is causing the pain. Workaround will be more cleaner if we can move balanced_irq_init() to late_initcall.

If there is no objection, I can post a new patch with that change.

Here is a new patch. Andrew please apply.

Thanks Suresh, looks good.

Zwane

#8 lore raw
In reply to Suresh Siddha
Quoted 4 lines · Suresh Siddha

drivers/pci/quirks.c line 823

Quoted 10 lines
drivers/pci/quirks.c +7 -0
@@ -814,6 +814,55 @@
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82801DB_12,	asus_hides_smbus_lpc );
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL,	PCI_DEVICE_ID_INTEL_82801EB_0,	asus_hides_smbus_lpc );
 
+#if defined(CONFIG_X86_IO_APIC) && defined(CONFIG_SMP)
+#include <asm/hw_irq.h>
+#ifdef CONFIG_IRQBALANCE
+extern int irqbalance_disable(char *str);
+#endif
+extern int no_irq_affinity;
+extern int noirqdebug_setup(char *str);

Please don't put extern declarations in .c files. Try to find a suitable header file which is included by both the definition and the users of these variables/functions.

If there is no appropriate header file, feel free to create a new one.

author
#9 lore raw
Quoted 1 line · Andrew Morton

Please don't put extern declarations in .c files.

Ok Andrew. Here is the new patch.

thanks,
suresh
--

As part of the workaround for the "Interrupt message re-ordering across hub interface" errata (page #16 in
http://developer.intel.com/design/chipsets/specupdt/30288402.pdf),
BIOS may enable hardware IRQ balancing for E7520/E7320/E7525(revision ID 0x9 and below) based platforms.

Add pci quirks to disable SW irqbalance/affinity on those platforms. Move balanced_irq_init() to late_initcall so that kirqd will be started after pci quirks.

Signed-off-by Suresh Siddha <[email protected]>
arch/i386/kernel/io_apic.c +2 -2
diff -Nru linux-2.6.9-rc3/arch/i386/kernel/io_apic.c linux-irq/arch/i386/kernel/io_apic.c
@@ -636,7 +636,7 @@
 	return 0;
 }
 
-static int __init irqbalance_disable(char *str)
+int __init irqbalance_disable(char *str)
 {
 	irqbalance_disabled = 1;
 	return 0;

@@ -653,7 +653,7 @@
 	}
 }
 
-__initcall(balanced_irq_init);
+late_initcall(balanced_irq_init);
 
 #else /* !CONFIG_IRQBALANCE */
 static inline void move_irq(int irq) { }
arch/i386/kernel/irq.c +4 -2
diff -Nru linux-2.6.9-rc3/arch/i386/kernel/irq.c linux-irq/arch/i386/kernel/irq.c
@@ -272,7 +272,7 @@
 
 static int noirqdebug;
 
-static int __init noirqdebug_setup(char *str)
+int __init noirqdebug_setup(char *str)
 {
 	noirqdebug = 1;
 	printk("IRQ lockup detection disabled\n");

@@ -997,13 +997,15 @@
 	return len;
 }
 
+int no_irq_affinity;
+
 static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
 					unsigned long count, void *data)
 {
 	int irq = (long)data, full_count = count, err;
 	cpumask_t new_value, tmp;
 
-	if (!irq_desc[irq].handler->set_affinity)
+	if (!irq_desc[irq].handler->set_affinity || no_irq_affinity)
 		return -EIO;
 
 	err = cpumask_parse(buffer, count, new_value);
arch/x86_64/kernel/irq.c +3 -1
diff -Nru linux-2.6.9-rc3/arch/x86_64/kernel/irq.c linux-irq/arch/x86_64/kernel/irq.c
@@ -835,6 +835,8 @@
 	return len;
 }
 
+int no_irq_affinity;
+
 static int irq_affinity_write_proc (struct file *file,
 					const char __user *buffer,
 					unsigned long count, void *data)

@@ -842,7 +844,7 @@
 	int irq = (long) data, full_count = count, err;
 	cpumask_t tmp, new_value;
 
-	if (!irq_desc[irq].handler->set_affinity)
+	if (!irq_desc[irq].handler->set_affinity || no_irq_affinity)
 		return -EIO;
 
 	err = cpumask_parse(buffer, count, new_value);
drivers/pci/quirks.c +43 -0

49 lines not on this page · raw message

include/asm-i386/irq.h +6 -0
diff -Nru linux-2.6.9-rc3/include/asm-i386/irq.h linux-irq/include/asm-i386/irq.h
@@ -55,4 +55,10 @@
 asmlinkage int handle_IRQ_event(unsigned int, struct pt_regs *,
 				struct irqaction *);
 
+#ifdef CONFIG_IRQBALANCE
+extern int irqbalance_disable(char *str);
+#endif
+extern int no_irq_affinity;
+extern int noirqdebug_setup(char *str);
+
 #endif /* _ASM_IRQ_H */
include/asm-x86_64/irq.h +2 -0
diff -Nru linux-2.6.9-rc3/include/asm-x86_64/irq.h linux-irq/include/asm-x86_64/irq.h
@@ -57,4 +57,6 @@
 struct pt_regs;
 int handle_IRQ_event(unsigned int, struct pt_regs *, struct irqaction *);
 
+extern int no_irq_affinity;
+
 #endif /* _ASM_IRQ_H */
include/linux/pci_ids.h +2 -0
diff -Nru linux-2.6.9-rc3/include/linux/pci_ids.h linux-irq/include/linux/pci_ids.h
@@ -2205,6 +2205,8 @@
 #define PCI_DEVICE_ID_INTEL_82855GM_HB	0x3580
 #define PCI_DEVICE_ID_INTEL_82855GM_IG	0x3582
 #define PCI_DEVICE_ID_INTEL_SMCH	0x3590
+#define PCI_DEVICE_ID_INTEL_E7320_MCH	0x3592
+#define PCI_DEVICE_ID_INTEL_E7525_MCH	0x359e
 #define PCI_DEVICE_ID_INTEL_80310	0x530d
 #define PCI_DEVICE_ID_INTEL_82371SB_0	0x7000
 #define PCI_DEVICE_ID_INTEL_82371SB_1	0x7010
#10 lore raw
In reply to Suresh Siddha
Quoted 7 lines · Suresh Siddha

As far as irq_affinity is concerned, workaround fits nicely in quirks.c infrastructure.

x86 irqbalance is the one which is causing the pain. Workaround will be more cleaner if we can move balanced_irq_init() to late_initcall.

If there is no objection, I can post a new patch with that change.

I don't see anything wrong with that, go ahead.

Thanks,
Zwane