Skip to contentSkip to conversation

MSI interrupts: disallow when no LAPIC/IOAPIC support

PATCH In review Randy.Dunlap 6 participants 11 messages Sep 27, 2005 – Sep 29, 2005 PCI

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 MSI interrupts: disallow when no LAPIC/IOAPIC support

From: Randy Dunlap <[email protected]>

MSI requires local APIC + IO APIC support (according to drivers/pci/Kconfig), but if a kernel is built with all of those (APICs + CONFIG_PCI_MSI) and then booted with "nosmp" or "max_cpus=0|1" or "noapic" or "nolapic", MSI also should be disabled, otherwise the interrupt routing is bad (so that only using "irqpoll" helps).

---

 arch/i386/kernel/io_apic.c   |    2 ++
 arch/i386/kernel/smpboot.c   |    6 +++++-
 arch/x86_64/kernel/apic.c    |    6 +++++-
 arch/x86_64/kernel/io_apic.c |    2 ++
 arch/x86_64/kernel/setup.c   |    4 +++-
 arch/x86_64/kernel/smpboot.c |   11 +++++++++--
 drivers/pci/msi.c            |    5 +++++
 include/linux/pci.h          |    2 ++
 8 files changed, 33 insertions(+), 5 deletions(-)
arch/i386/kernel/io_apic.c +2 -0
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/i386/kernel/io_apic.c linux-2614-rc2-git4/arch/i386/kernel/io_apic.c
@@ -31,6 +31,7 @@
 #include <linux/mc146818rtc.h>
 #include <linux/compiler.h>
 #include <linux/acpi.h>
+#include <linux/pci.h>
 #include <linux/module.h>
 #include <linux/sysdev.h>
 
@@ -684,6 +685,7 @@ int skip_ioapic_setup;
 static int __init ioapic_setup(char *str)
 {
 	skip_ioapic_setup = 1;
+	msi_off();
 	return 1;
 }
 
arch/i386/kernel/smpboot.c +5 -1
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/i386/kernel/smpboot.c linux-2614-rc2-git4/arch/i386/kernel/smpboot.c
@@ -46,6 +46,7 @@
 #include <linux/bootmem.h>
 #include <linux/notifier.h>
 #include <linux/cpu.h>
+#include <linux/pci.h>
 #include <linux/percpu.h>
 
 #include <linux/delay.h>
@@ -1107,9 +1108,11 @@ static void __init smp_boot_cpus(unsigne
 		printk(KERN_NOTICE "SMP motherboard not detected.\n");
 		smpboot_clear_io_apic_irqs();
 		phys_cpu_present_map = physid_mask_of_physid(0);
-		if (APIC_init_uniprocessor())
+		if (APIC_init_uniprocessor()) {
 			printk(KERN_NOTICE "Local APIC not detected."
 					   " Using dummy APIC emulation.\n");
+			msi_off();
+		}
 		map_cpu_to_logical_apicid();
 		cpu_set(0, cpu_sibling_map[0]);
 		cpu_set(0, cpu_core_map[0]);
@@ -1150,6 +1153,7 @@ static void __init smp_boot_cpus(unsigne
 		smp_found_config = 0;
 		printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
 		smpboot_clear_io_apic_irqs();
+		msi_off();
 		phys_cpu_present_map = physid_mask_of_physid(0);
 		cpu_set(0, cpu_sibling_map[0]);
 		cpu_set(0, cpu_core_map[0]);
arch/x86_64/kernel/apic.c +5 -1
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/apic.c linux-2614-rc2-git4/arch/x86_64/kernel/apic.c
@@ -24,6 +24,7 @@
 #include <linux/interrupt.h>
 #include <linux/mc146818rtc.h>
 #include <linux/kernel_stat.h>
+#include <linux/pci.h>
 #include <linux/sysdev.h>
 
 #include <asm/atomic.h>
@@ -1057,8 +1058,10 @@ int __init APIC_init_uniprocessor (void)
 #ifdef CONFIG_X86_IO_APIC
 	if (smp_found_config && !skip_ioapic_setup && nr_ioapics)
 			setup_IO_APIC();
-	else
+	else {
 		nr_ioapics = 0;
+		msi_off();
+	}
 #endif
 	setup_boot_APIC_clock();
 	check_nmi_watchdog();
@@ -1074,6 +1077,7 @@ static __init int setup_disableapic(char
 static __init int setup_nolapic(char *str) 
 { 
 	disable_apic = 1;
+	msi_off();
 	return 0;
 } 
 
arch/x86_64/kernel/io_apic.c +2 -0
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/io_apic.c linux-2614-rc2-git4/arch/x86_64/kernel/io_apic.c
@@ -29,6 +29,7 @@
 #include <linux/smp_lock.h>
 #include <linux/mc146818rtc.h>
 #include <linux/acpi.h>
+#include <linux/pci.h>
 #include <linux/sysdev.h>
 
 #include <asm/io.h>
@@ -227,6 +228,7 @@ int ioapic_force;
 static int __init disable_ioapic_setup(char *str)
 {
 	skip_ioapic_setup = 1;
+	msi_off();
 	return 1;
 }
 
arch/x86_64/kernel/setup.c +3 -1
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/setup.c linux-2614-rc2-git4/arch/x86_64/kernel/setup.c
@@ -345,8 +345,10 @@ static __init void parse_cmdline_early (
 		    !memcmp(from, "disableapic", 11))
 			disable_apic = 1;
 
-		if (!memcmp(from, "noapic", 6)) 
+		if (!memcmp(from, "noapic", 6)) {
 			skip_ioapic_setup = 1;
+			msi_off();
+		}
 
 		if (!memcmp(from, "apic", 4)) { 
 			skip_ioapic_setup = 0;
arch/x86_64/kernel/smpboot.c +9 -2
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/arch/x86_64/kernel/smpboot.c linux-2614-rc2-git4/arch/x86_64/kernel/smpboot.c
@@ -47,6 +47,7 @@
 #include <linux/bootmem.h>
 #include <linux/thread_info.h>
 #include <linux/module.h>
+#include <linux/pci.h>
 
 #include <linux/delay.h>
 #include <linux/mc146818rtc.h>
@@ -918,9 +919,11 @@ static int __init smp_sanity_check(unsig
 	if (!smp_found_config) {
 		printk(KERN_NOTICE "SMP motherboard not detected.\n");
 		disable_smp();
-		if (APIC_init_uniprocessor())
+		if (APIC_init_uniprocessor()) {
 			printk(KERN_NOTICE "Local APIC not detected."
 					   " Using dummy APIC emulation.\n");
+			msi_off();
+		}
 		return -1;
 	}
 
@@ -942,6 +945,7 @@ static int __init smp_sanity_check(unsig
 			boot_cpu_id);
 		printk(KERN_ERR "... forcing use of dummy APIC emulation. (tell your hw vendor)\n");
 		nr_ioapics = 0;
+		msi_off();
 		return -1;
 	}
 
@@ -951,6 +955,7 @@ static int __init smp_sanity_check(unsig
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated, forcing use of dummy APIC emulation.\n");
 		nr_ioapics = 0;
+		msi_off();
 		return -1;
 	}
 
@@ -995,8 +1000,10 @@ void __init smp_prepare_cpus(unsigned in
 	 */
 	if (!skip_ioapic_setup && nr_ioapics)
 		setup_IO_APIC();
-	else
+	else {
 		nr_ioapics = 0;
+		msi_off();
+	}
 
 	/*
 	 * Set up local APIC timer on boot CPU.
drivers/pci/msi.c +5 -0
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/drivers/pci/msi.c linux-2614-rc2-git4/drivers/pci/msi.c
@@ -1113,6 +1113,11 @@ void msi_remove_pci_irq_vectors(struct p
 	}
 }
 
+void msi_off(void)
+{
+	pci_msi_enable = 0;
+}
+
 EXPORT_SYMBOL(pci_enable_msi);
 EXPORT_SYMBOL(pci_disable_msi);
 EXPORT_SYMBOL(pci_enable_msix);
include/linux/pci.h +2 -0
diff -Naurp -X linux-2614-rc2/Documentation/dontdiff linux-2614-rc2-git4-pv/include/linux/pci.h linux-2614-rc2-git4/include/linux/pci.h
@@ -480,6 +480,7 @@ static inline int pci_enable_msix(struct
 	struct msix_entry *entries, int nvec) {return -1;}
 static inline void pci_disable_msix(struct pci_dev *dev) {}
 static inline void msi_remove_pci_irq_vectors(struct pci_dev *dev) {}
+static inline void msi_off(void) {}
 #else
 extern void pci_scan_msi_device(struct pci_dev *dev);
 extern int pci_enable_msi(struct pci_dev *dev);
@@ -488,6 +489,7 @@ extern int pci_enable_msix(struct pci_de
 	struct msix_entry *entries, int nvec);
 extern void pci_disable_msix(struct pci_dev *dev);
 extern void msi_remove_pci_irq_vectors(struct pci_dev *dev);
+extern void msi_off(void);
 #endif
 
 /*

---

#1 lore raw
Quoted 4 lines · Randy.Dunlap

From: Randy Dunlap <[email protected]>

MSI requires local APIC + IO APIC support (according to drivers/pci/Kconfig),

The purpose of MSI is to bypass the IO APIC. I'm not aware of any dependency on IO APIC in HW. I suspect the dependency in the Kconfig is historical because of how the x86 source code is structured:
config X86_LOCAL_APIC

        bool
        depends on X86_UP_APIC || ((X86_VISWS || SMP) && !X86_VOYAGER)
        default y

essentially, LOCAL_APIC gets enabled if "UP_APIC or SMP". I've no clue why folks thought it was better to ignore the IO APIC on UP kernels.

I'm pretty sure one only needs a local APIC support. Local APIC responds to 0xfee00000 range of addresses and directs the interrupt to a CPU.

Quoted 5 lines

but if a kernel is built with all of
those (APICs + CONFIG_PCI_MSI) and then booted with "nosmp" or "max_cpus=0|1" or "noapic" or "nolapic", MSI also should be disabled, otherwise the interrupt routing is bad (so that only using "irqpoll" helps).

UP vs SMP is orthogonal to MSI support wrt HW.
I suppose this is another historical artifact of APICs being used with SMP x86 boxes first.

thanks,
grant

#2 lore raw
Quoted 15 lines · On Mon, 26 Sep 2005 22:48:40 -0600 Grant Grundler wrote:
Quoted 4 lines · Randy.Dunlap

From: Randy Dunlap <[email protected]>

MSI requires local APIC + IO APIC support (according to drivers/pci/Kconfig),

The purpose of MSI is to bypass the IO APIC. I'm not aware of any dependency on IO APIC in HW. I suspect the dependency in the Kconfig is historical because of how the x86 source code is structured:
config X86_LOCAL_APIC

        bool
        depends on X86_UP_APIC || ((X86_VISWS || SMP) && !X86_VOYAGER)
        default y
Quoted 7 lines

essentially, LOCAL_APIC gets enabled if "UP_APIC or SMP". I've no clue why folks thought it was better to ignore the IO APIC on UP kernels.

I'm pretty sure one only needs a local APIC support. Local APIC responds to 0xfee00000 range of addresses and directs the interrupt to a CPU.

Thanks. Can't say I'm surprised at that info.

Quoted 9 lines
Quoted 5 lines

but if a kernel is built with all of
those (APICs + CONFIG_PCI_MSI) and then booted with "nosmp" or "max_cpus=0|1" or "noapic" or "nolapic", MSI also should be disabled, otherwise the interrupt routing is bad (so that only using "irqpoll" helps).

UP vs SMP is orthogonal to MSI support wrt HW.
I suppose this is another historical artifact of APICs being used with SMP x86 boxes first.

"nosmp" (currently) means 1 CPU and no LAPICs/no IOAPICs. It doesn't have to be that way, but yes, I suspect that it's mostly historical, plus a method of getting to a lowest common hardware boot sequence, which is sometimes nice for debugging or installation.

Nevertheless, there is a problem here. What do you suggest to solve it? Just making PCI_MSI depend on Local APIC support, or something else?

[Kernel is assigning MSI interrupts, but then they are "lost." Using "irqpoll" will find them, but that's a performance penalty.]

---
~Randy
You can't do anything without having to do something else first. -- Belefant's Law

#3 lore raw
Quoted 1 line · Randy.Dunlap

"nosmp" (currently) means 1 CPU and no LAPICs/no IOAPICs.

ok - ie we ignore HW that may be present.

In a way, this makes sense since LAPIC was used for IRQ "load balancing" - ie IRQs could get redirected to another "less busy" CPU. IIRC, discussions about "XPR" should explain how that works.

Quoted 8 lines

It doesn't have to be that way, but yes, I suspect that it's mostly historical, plus a method of getting to a lowest common hardware boot sequence, which is sometimes nice for debugging or installation.

Nevertheless, there is a problem here. What do you suggest to solve it? Just making PCI_MSI depend on Local APIC support, or something else?

Yeah, my preference would be PCI_MSI only depend on Local APIC. (Note that having a Local APIC implies having an IO APIC as well
- but MSI should completely ignore it.)

But I don't know enough x86 history to know if thats feasible or not. And I'm not blessed with the time to unravel the x86 APIC support - if the dependencies are necessary, so be it.

Quoted 2 lines

[Kernel is assigning MSI interrupts, but then they are "lost." Using "irqpoll" will find them, but that's a performance penalty.]

Interesting. I'm suprised an MSI can get "lost".
It implies MSI code is allocating a CPU vector but the vector is not getting enabled/unmasked or the Local APIC is ignoring it.

Yeah, we don't want to be using irqpoll for this - entirely defeats the purpose of MSI.

thanks,
grant

#4 lore raw
Quoted 2 lines · Grant Grundler

Yeah, my preference would be PCI_MSI only depend on Local APIC. (Note that having a Local APIC implies having an IO APIC as well

Not true. Lots of systems have a local APIC but no I/O APICs. However, having an I/O APIC pretty much implies having a local APIC.

#5 lore raw
Quoted 5 lines · Mikael Pettersson
Quoted 2 lines · Grant Grundler

Yeah, my preference would be PCI_MSI only depend on Local APIC. (Note that having a Local APIC implies having an IO APIC as well

Not true. Lots of systems have a local APIC but no I/O APICs.

Eh?!!!
Do systems with Local APIC still have IRQ lines going to the CPU? How does PCI IRQ line otherwise generate an interrupt?

Quoted 1 line

However, having an I/O APIC pretty much implies having a local APIC.

Ok - That sounds right for x86 and is true for ia64. parisc has IO SAPICs but no local APIC.

parisc and (IIRC) alpha has the functional equivalent of Local APIC but the implementation details are different. ie they can't share code with x86/ia64.

thanks,
grant

#6 lore raw
In reply to Randy.Dunlap
Quoted 1 line · Randy.Dunlap

"nosmp" (currently) means 1 CPU and no LAPICs/no IOAPICs.

nosmp shouldn't disable the local APIC or the I/O APIC(s). If it really does, then that's a bug. We have noapic/nolapic/lapic to control enablement of the APICs.

#7 lore raw
In reply to Grant Grundler
Quoted 2 lines · Grant Grundler

I've no clue why folks thought it was better to ignore the IO APIC on UP kernels.

Hysterical raisins: the -majority- of the early uniprocessor systems that claimed IOAPIC support were broken.

Jeff

#8 lore raw
Quoted 6 lines · Jeff Garzik
Quoted 2 lines · Grant Grundler

I've no clue why folks thought it was better to ignore the IO APIC on UP kernels.

Hysterical raisins: the -majority- of the early uniprocessor systems that claimed IOAPIC support were broken.

Not really broken in most cases, but since nobody was using the APIC board makers didn't bother wiring for it.

#9 lore raw
Quoted 10 lines · Alan Cox
Quoted 6 lines · Jeff Garzik
Quoted 2 lines · Grant Grundler

I've no clue why folks thought it was better to ignore the IO APIC on UP kernels.

Hysterical raisins: the -majority- of the early uniprocessor systems that claimed IOAPIC support were broken.

Not really broken in most cases, but since nobody was using the APIC board makers didn't bother wiring for it.

ok. Any clue how PCI IRQs got routed/handled on those boxes? Did UP boards have an 8259 PIC and an IRQ line to the CPU? Could an 8529 PIC even co-exist with an IO APIC?

Or was it something more silly like BIOS mfgs had no version of Windows that could grok a IRQ routing table and thus no incentive to enable that feature?

thanks,
grant

#10 lore raw
Quoted 9 lines · Grant Grundler
Quoted 8 lines · Alan Cox
Quoted 6 lines · Jeff Garzik
Quoted 2 lines · Grant Grundler

I've no clue why folks thought it was better to ignore the IO APIC on UP kernels.

Hysterical raisins: the -majority- of the early uniprocessor systems that claimed IOAPIC support were broken.

Quoted 6 lines
Quoted 2 lines

Not really broken in most cases, but since nobody was using the APIC board makers didn't bother wiring for it.

ok. Any clue how PCI IRQs got routed/handled on those boxes? Did UP boards have an 8259 PIC and an IRQ line to the CPU? Could an 8529 PIC even co-exist with an IO APIC?

x86 linux can't even boot without using an 8259 PIC. The 8259 PIC if you don't have an IOAPIC is wired to the local apic and the appropriate pin is placed in ExtINT mode.

Most current systems have an 8259 PIC, their primary PIC, and a PIRQ router all in the same chip.

Eric