Skip to contentSkip to conversation

IRQ compression/sharing patch

RFC In review James Cleverdon 3 participants 11 messages Jul 26, 2005 – Aug 16, 2005

lore
Share & tools
lore
#0 lore raw

RFC 1/1 IRQ compression/sharing patch

Here's a patch that builds on Natalie Protasevich's IRQ compression patch and tries to work for MPS boots as well as ACPI. It is meant for a 4-node IBM x460 NUMA box, which was dying because it had interrupt pins with GSI numbers > NR_IRQS and thus overflowed irq_desc.

The problem is that this system has 270 GSIs (which are 1:1 mapped with I/O APIC RTEs) and an 8-node box would have 540. This is much bigger than NR_IRQS (224 for both i386 and x86_64). Also, there aren't enough vectors to go around. There are about 190 usable vectors, not counting the reserved ones and the unused vectors at 0x20 to 0x2F. So, my patch attempts to compress the GSI range and share vectors by sharing IRQs.

Important safety note: While the SLES 9 version of this patch works, I haven't been able to test the -rc3-mm1 patch enclosed. I keep getting errors from the adp94xx driver. 8-(

(Sorry about doing an attachment, but KMail is steadfastly word wrapping inserted files. I need to upgrade....)

Signature · 3 lines
James Cleverdon
IBM LTC (xSeries Linux Solutions)
{jamesclv(Unix, preferred), cleverdj(Notes)} at us dot ibm dot comm
arch/i386/kernel/acpi/boot.c +7 -2
diff -pru 2.6.13-rc3-mm1/arch/i386/kernel/acpi/boot.c j13-rc3-mm1/arch/i386/kernel/acpi/boot.c
@@ -40,9 +40,10 @@
 
 #ifdef	CONFIG_X86_64
 
-static inline void  acpi_madt_oem_check(char *oem_id, char *oem_table_id) { }
+extern void  acpi_madt_oem_check(char *oem_id, char *oem_table_id);
 extern void __init clustered_apic_check(void);
 static inline int ioapic_setup_disabled(void) { return 0; }
+extern int gsi_irq_sharing(int gsi);
 #include <asm/proto.h>
 
 #else	/* X86 */
@@ -52,6 +53,10 @@ static inline int ioapic_setup_disabled(
 #include <mach_mpparse.h>
 #endif	/* CONFIG_X86_LOCAL_APIC */
 
+static inline int ioapic_setup_disabled(void) { return 0; }
+static inline int gsi_irq_sharing(int gsi) { return gsi; }
+
+
 #endif	/* X86 */
 
 #define BAD_MADT_ENTRY(entry, end) (					    \
@@ -480,7 +485,7 @@ int acpi_gsi_to_irq(u32 gsi, unsigned in
  		*irq = IO_APIC_VECTOR(gsi);
 	else
 #endif
-		*irq = gsi;
+		*irq = gsi_irq_sharing(gsi);
 	return 0;
 }
 
arch/x86_64/Kconfig +4 -4
diff -pru 2.6.13-rc3-mm1/arch/x86_64/Kconfig j13-rc3-mm1/arch/x86_64/Kconfig
@@ -276,13 +276,13 @@ config HAVE_DEC_LOCK
 	default y
 
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "64"
 	help
 	  This allows you to specify the maximum number of CPUs which this
-	  kernel will support. Current maximum is 256 CPUs due to
+	  kernel will support. Current maximum is 255 CPUs due to
 	  APIC addressing limits. Less depending on the hardware.
 
 	  This is purely to save memory - each supported CPU requires
arch/x86_64/kernel/genapic.c +16 -0
diff -pru 2.6.13-rc3-mm1/arch/x86_64/kernel/genapic.c j13-rc3-mm1/arch/x86_64/kernel/genapic.c
@@ -103,3 +103,19 @@ void send_IPI_self(int vector)
 {
 	__send_IPI_shortcut(APIC_DEST_SELF, vector, APIC_DEST_PHYSICAL);
 }
+
+/*
+ * Check the APIC IDs in MADT table header and choose the APIC mode.
+ */
+void acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+{
+	/* May need to check OEM strings in the future. */
+}
+
+/*
+ * Check the IDs in MPS header and choose the APIC mode.
+ */
+void mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid)
+{
+	/* May need to check OEM strings in the future. */
+}
arch/x86_64/kernel/io_apic.c +73 -7

158 lines not on this page · raw message

arch/x86_64/kernel/irq.c +1 -1
diff -pru 2.6.13-rc3-mm1/arch/x86_64/kernel/irq.c j13-rc3-mm1/arch/x86_64/kernel/irq.c
@@ -99,7 +99,7 @@ asmlinkage unsigned int do_IRQ(struct pt
 	unsigned irq = regs->orig_rax & 0xff;
 
 	irq_enter();
-	BUG_ON(irq > 256);
+	BUG_ON(irq > NR_IRQS);
 
 	__do_IRQ(irq, regs);
 	irq_exit();
arch/x86_64/kernel/mpparse.c +8 -5
diff -pru 2.6.13-rc3-mm1/arch/x86_64/kernel/mpparse.c j13-rc3-mm1/arch/x86_64/kernel/mpparse.c
@@ -216,7 +216,7 @@ static void __init MP_intsrc_info (struc
 			m->mpc_irqtype, m->mpc_irqflag & 3,
 			(m->mpc_irqflag >> 2) & 3, m->mpc_srcbus,
 			m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq);
-	if (++mp_irq_entries == MAX_IRQ_SOURCES)
+	if (++mp_irq_entries >= MAX_IRQ_SOURCES)
 		panic("Max # of irq sources exceeded!!\n");
 }
 
@@ -248,9 +248,10 @@ static void __init MP_lintsrc_info (stru
 
 static int __init smp_read_mpc(struct mp_config_table *mpc)
 {
-	char str[16];
+	char oem[9], str[16];
 	int count=sizeof(*mpc);
 	unsigned char *mpt=((unsigned char *)mpc)+count;
+	extern void mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid);
 
 	if (memcmp(mpc->mpc_signature,MPC_SIGNATURE,4)) {
 		printk("SMP mptable: bad signature [%c%c%c%c]!\n",
@@ -273,14 +274,16 @@ static int __init smp_read_mpc(struct mp
 		printk(KERN_ERR "SMP mptable: null local APIC address!\n");
 		return 0;
 	}
-	memcpy(str,mpc->mpc_oem,8);
-	str[8]=0;
-	printk(KERN_INFO "OEM ID: %s ",str);
+	memcpy(oem, mpc->mpc_oem, 8);
+	oem[8] = 0;
+	printk(KERN_INFO "OEM ID: %s ", oem);
 
 	memcpy(str,mpc->mpc_productid,12);
 	str[12]=0;
 	printk(KERN_INFO "Product ID: %s ",str);
 
+	mps_oem_check(mpc, oem, str);
+
 	printk(KERN_INFO "APIC at: 0x%X\n",mpc->mpc_lapic);
 
 	/* save the local APIC address, it might be non-default */
include/asm-x86_64/desc.h +3 -0
diff -pru 2.6.13-rc3-mm1/include/asm-x86_64/desc.h j13-rc3-mm1/include/asm-x86_64/desc.h
@@ -95,16 +95,19 @@ static inline void _set_gate(void *adr, 
 
 static inline void set_intr_gate(int nr, void *func) 
 { 
+	BUG_ON((unsigned)nr > 0xFF);
 	_set_gate(&idt_table[nr], GATE_INTERRUPT, (unsigned long) func, 0, 0); 
 } 
 
 static inline void set_intr_gate_ist(int nr, void *func, unsigned ist) 
 { 
+	BUG_ON((unsigned)nr > 0xFF);
 	_set_gate(&idt_table[nr], GATE_INTERRUPT, (unsigned long) func, 0, ist); 
 } 
 
 static inline void set_system_gate(int nr, void *func) 
 { 
+	BUG_ON((unsigned)nr > 0xFF);
 	_set_gate(&idt_table[nr], GATE_INTERRUPT, (unsigned long) func, 3, 0); 
 } 
 
include/asm-x86_64/mpspec.h +2 -1
diff -pru 2.6.13-rc3-mm1/include/asm-x86_64/mpspec.h j13-rc3-mm1/include/asm-x86_64/mpspec.h
@@ -157,7 +157,8 @@ struct mpc_config_lintsrc
  */
 
 #define MAX_MP_BUSSES 256
-#define MAX_IRQ_SOURCES 256
+/* Each PCI slot may be a combo card with its own bus.  4 IRQ pins per slot. */
+#define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4)
 enum mp_bustype {
 	MP_BUS_ISA = 1,
 	MP_BUS_EISA,
#1 lore raw
Quoted 12 lines · James Cleverdon

Here's a patch that builds on Natalie Protasevich's IRQ compression patch and tries to work for MPS boots as well as ACPI. It is meant for a 4-node IBM x460 NUMA box, which was dying because it had interrupt pins with GSI numbers > NR_IRQS and thus overflowed irq_desc.

The problem is that this system has 270 GSIs (which are 1:1 mapped with I/O APIC RTEs) and an 8-node box would have 540. This is much bigger than NR_IRQS (224 for both i386 and x86_64). Also, there aren't enough vectors to go around. There are about 190 usable vectors, not counting the reserved ones and the unused vectors at 0x20 to 0x2F. So, my patch attempts to compress the GSI range and share vectors by sharing IRQs.

Quoted 6 lines

Important safety note: While the SLES 9 version of this patch works, I haven't been able to test the -rc3-mm1 patch enclosed. I keep getting errors from the adp94xx driver. 8-(

(Sorry about doing an attachment, but KMail is steadfastly word wrapping inserted files. I need to upgrade....)

The patch seems to have lots of unrelated stuff. Can you please split it out?

BTW I plan to implement per CPU IDT vectors similar to Zwane's i386 patch for x86-64 soon, hopefully with that things will be easier too.

Andrew: this is not 2.6.13 material.

arch/x86_64/Kconfig config HAVE_DEC_LOCK · line 282

Quoted 11 lines
arch/x86_64/Kconfig +3 -3
@@ -276,13 +276,13 @@ config HAVE_DEC_LOCK
 	default y
 
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "64"

Please don't change that,

Quoted 5 lines
arch/x86_64/kernel/genapic.c +5 -0
+/*
+ * Check the APIC IDs in MADT table header and choose the APIC mode.
+ */
+void acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+{

arch/x86_64/kernel/genapic.c void send_IPI_self(int vector) · line 121

Quoted 10 lines
arch/x86_64/kernel/genapic.c +10 -0
+	/* May need to check OEM strings in the future. */
+}
+
+/*
+ * Check the IDs in MPS header and choose the APIC mode.
+ */
+void mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid)
+{
+	/* May need to check OEM strings in the future. */
+}

Can you perhaps add it then later, not now?

Quoted 1 line

+static u8 gsi_2_irq[NR_IRQ_VECTORS] = { [0 ... NR_IRQ_VECTORS-1] = 0xFF };

With the per cpu IDTs we'll likely need more than 8 bits here.

arch/x86_64/kernel/mpparse.c static void __init MP_lintsrc_info (stru · line 254

Quoted 5 lines
arch/x86_64/kernel/mpparse.c +2 -1
-	char str[16];
+	char oem[9], str[16];
 	int count=sizeof(*mpc);
 	unsigned char *mpt=((unsigned char *)mpc)+count;
+	extern void mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid);

That would belong in some header if it was needed.

But please just remove it for now.

The rest looks ok.

-Andi

#2 lore raw
Quoted 19 lines · Andi Kleen
Quoted 18 lines · James Cleverdon

Here's a patch that builds on Natalie Protasevich's IRQ compression patch and tries to work for MPS boots as well as ACPI. It is meant for a 4-node IBM x460 NUMA box, which was dying because it had interrupt pins with GSI numbers > NR_IRQS and thus overflowed irq_desc.

The problem is that this system has 270 GSIs (which are 1:1 mapped with I/O APIC RTEs) and an 8-node box would have 540. This is much bigger than NR_IRQS (224 for both i386 and x86_64). Also, there aren't enough vectors to go around. There are about 190 usable vectors, not counting the reserved ones and the unused vectors at 0x20 to 0x2F. So, my patch attempts to compress the GSI range and share vectors by sharing IRQs.

Important safety note: While the SLES 9 version of this patch works, I haven't been able to test the -rc3-mm1 patch enclosed. I keep getting errors from the adp94xx driver. 8-(

Quoted 5 lines
Quoted 2 lines

(Sorry about doing an attachment, but KMail is steadfastly word wrapping inserted files. I need to upgrade....)

The patch seems to have lots of unrelated stuff. Can you please split it out?

Of course. I'll pull out the BUG_ON()s and some of the other cleanup stuff into a related patch.

Quoted 2 lines

BTW I plan to implement per CPU IDT vectors similar to Zwane's i386 patch for x86-64 soon, hopefully with that things will be easier too.

I hope so, too. The problem has been making the most of limited interrupt resources (vectors and IRQ numbers), when previous coding has assumed that we could use them lavishly.

Quoted 5 lines

Andrew: this is not 2.6.13 material.

Quoted 3 lines
arch/x86_64/Kconfig +0 -0
@@ -276,13 +276,13 @@ config HAVE_DEC_LOCK
 	default y
 
Quoted 10 lines

arch/x86_64/Kconfig config HAVE_DEC_LOCK · line 282

Quoted 8 lines
arch/x86_64/Kconfig +3 -3
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "64"

Please don't change that,

Which? The maximum number of addressable CPUs is 255, because FF is reserved. Or, would you rather the default be 8 or 16? (Hmmm.... Dual-core, hyperthreaded CPUs are out. They'll turn a quad box into a 16-way.)

Quoted 9 lines
Quoted 9 lines
(quoted diff) +2 -0
+/*
+ * Check the APIC IDs in MADT table header and choose the APIC

mode. + */

arch/x86_64/kernel/genapic.c +6 -0
+void acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+{
+	/* May need to check OEM strings in the future. */
+}
+
+/*
Quoted 8 lines

Review

Quoted 6 lines

+ * Check the IDs in MPS header and choose the APIC mode.
+ */
+void mps_oem_check(struct mp_config_table *mpc, char *oem, char *productid) +{

(quoted diff) +2 -0
+	/* May need to check OEM strings in the future. */
+}

Can you perhaps add it then later, not now?

Naturally. It was a placeholder for those systems where we can't figure out what to do using heuristics on the ACPI/MPS table data.

Quoted 4 lines
Quoted 2 lines

+static u8 gsi_2_irq[NR_IRQ_VECTORS] = { [0 ... NR_IRQ_VECTORS-1] = 0xFF };

With the per cpu IDTs we'll likely need more than 8 bits here.

OK, I'll make it u32. Or would you rather have u16?

Quoted 10 lines
Quoted 6 lines
arch/x86_64/kernel/mpparse.c +2 -1
-	char str[16];
+	char oem[9], str[16];
 	int count=sizeof(*mpc);
 	unsigned char *mpt=((unsigned char *)mpc)+count;
+	extern void mps_oem_check(struct mp_config_table *mpc, char *oem,

char *productid);

That would belong in some header if it was needed.

But please just remove it for now.

Right.

Quoted 3 lines

The rest looks ok.

-Andi

Thanks Andi!

Signature · 3 lines
James Cleverdon
IBM LTC (xSeries Linux Solutions)
{jamesclv(Unix, preferred), cleverdj(Notes)} at us dot ibm dot comm
#3 lore raw

RFC 1/1 [2.6.12.3] IRQ compression/sharing patch

In reply to Andi Kleen

Due to some device driver issues, I built this iteration of the patch vs. 2.6.12.3.

(Sorry about the attachment, but KMail is still word wrapping inserted files.)

Background:

Here's a patch that builds on Natalie Protasevich's IRQ compression patch and tries to work for MPS boots as well as ACPI. It is meant for a 4-node IBM x460 NUMA box, which was dying because it had interrupt pins with GSI numbers > NR_IRQS and thus overflowed irq_desc.

The problem is that this system has 280 GSIs (which are 1:1 mapped with I/O APIC RTEs) and an 8-node box would have 560. This is much bigger than NR_IRQS (224 for both i386 and x86_64). Also, there aren't enough vectors to go around. There are about 190 usable vectors, not counting the reserved ones and the unused vectors at 0x20 to 0x2F. So, my patch attempts to compress the GSI range and share vectors by sharing IRQs.

Signature · 3 lines
James Cleverdon
IBM LTC (xSeries Linux Solutions)
{jamesclv(Unix, preferred), cleverdj(Notes)} at us dot ibm dot comm
arch/i386/kernel/acpi/boot.c +5 -1
diff -pruN 2.6.12.3/arch/i386/kernel/acpi/boot.c n12.3/arch/i386/kernel/acpi/boot.c
@@ -42,6 +42,7 @@
 static inline void  acpi_madt_oem_check(char *oem_id, char *oem_table_id) { }
 extern void __init clustered_apic_check(void);
 static inline int ioapic_setup_disabled(void) { return 0; }
+extern int gsi_irq_sharing(int gsi);
 #include <asm/proto.h>
 
 #else	/* X86 */
@@ -51,6 +52,9 @@ static inline int ioapic_setup_disabled(
 #include <mach_mpparse.h>
 #endif	/* CONFIG_X86_LOCAL_APIC */
 
+static inline int gsi_irq_sharing(int gsi) { return gsi; }
+
+
 #endif	/* X86 */
 
 #define BAD_MADT_ENTRY(entry, end) (					    \
@@ -453,7 +457,7 @@ int acpi_gsi_to_irq(u32 gsi, unsigned in
  		*irq = IO_APIC_VECTOR(gsi);
 	else
 #endif
-		*irq = gsi;
+		*irq = gsi_irq_sharing(gsi);
 	return 0;
 }
 
arch/x86_64/Kconfig +4 -4
diff -pruN 2.6.12.3/arch/x86_64/Kconfig n12.3/arch/x86_64/Kconfig
@@ -280,13 +280,13 @@ config HAVE_DEC_LOCK
 	default y
 
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "16"
 	help
 	  This allows you to specify the maximum number of CPUs which this
-	  kernel will support. Current maximum is 256 CPUs due to
+	  kernel will support. Current maximum is 255 CPUs due to
 	  APIC addressing limits. Less depending on the hardware.
 
 	  This is purely to save memory - each supported CPU requires
arch/x86_64/kernel/io_apic.c +65 -4

105 lines not on this page · raw message

arch/x86_64/kernel/mpparse.c +1 -1
diff -pruN 2.6.12.3/arch/x86_64/kernel/mpparse.c n12.3/arch/x86_64/kernel/mpparse.c
@@ -214,7 +214,7 @@ static void __init MP_intsrc_info (struc
 			m->mpc_irqtype, m->mpc_irqflag & 3,
 			(m->mpc_irqflag >> 2) & 3, m->mpc_srcbus,
 			m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq);
-	if (++mp_irq_entries == MAX_IRQ_SOURCES)
+	if (++mp_irq_entries >= MAX_IRQ_SOURCES)
 		panic("Max # of irq sources exceeded!!\n");
 }
 
include/asm-x86_64/mpspec.h +2 -1
diff -pruN 2.6.12.3/include/asm-x86_64/mpspec.h n12.3/include/asm-x86_64/mpspec.h
@@ -157,7 +157,8 @@ struct mpc_config_lintsrc
  */
 
 #define MAX_MP_BUSSES 256
-#define MAX_IRQ_SOURCES 256
+/* Each PCI slot may be a combo card with its own bus.  4 IRQ pins per slot. */
+#define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4)
 enum mp_bustype {
 	MP_BUS_ISA = 1,
 	MP_BUS_EISA,
#4 lore raw

Re: [RFC][2.6.12.3] IRQ compression/sharing patch

Quoted 12 lines · James Cleverdon

diff -pruN 2.6.12.3/arch/i386/kernel/acpi/boot.c n12.3/arch/i386/kernel/acpi/boot.c

arch/i386/kernel/acpi/boot.c +1 -0
@@ -42,6 +42,7 @@
 static inline void  acpi_madt_oem_check(char *oem_id, char *oem_table_id) { }
 extern void __init clustered_apic_check(void);
 static inline int ioapic_setup_disabled(void) { return 0; }
+extern int gsi_irq_sharing(int gsi);
 #include <asm/proto.h>
 
 #else	/* X86 */

arch/i386/kernel/acpi/boot.c static inline int ioapic_setup_disabled( · line 55

Quoted 4 lines
arch/i386/kernel/acpi/boot.c +1 -0
@@ -51,6 +52,9 @@ static inline int ioapic_setup_disabled(
 #include <mach_mpparse.h>
 #endif	/* CONFIG_X86_LOCAL_APIC */
 
+static inline int gsi_irq_sharing(int gsi) { return gsi; }

Why is this different for i386/x86-64? It shouldn't.

As a unrelated note we really need to get rid of this whole ifdef block.

arch/i386/kernel/acpi/boot.c config HAVE_DEC_LOCK · line 286

Quoted 12 lines
arch/i386/kernel/acpi/boot.c +4 -3
+++ n12.3/arch/x86_64/Kconfig	2005-08-03 21:31:07.487451167 -0700
@@ -280,13 +280,13 @@ config HAVE_DEC_LOCK
 	default y
 
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "16"

Don't change the default please.

arch/i386/kernel/acpi/boot.c config HAVE_DEC_LOCK · line 287

Quoted 1 line
arch/i386/kernel/acpi/boot.c +1 -0
+static int next_irq = 16;

Won't this need a lock for hotplug later?

arch/i386/kernel/acpi/boot.c config HAVE_DEC_LOCK

Quoted 10 lines
arch/i386/kernel/acpi/boot.c +10 -0
+
+ retry_vector:
+	vector = assign_irq_vector(gsi);
+
+	/*
+	 * Sharing vectors means sharing IRQs, so scan irq_vectors for previous
+	 * use of vector and if found, return that IRQ.  However, we never want
+	 * to share legacy IRQs, which usually have a different trigger mode
+	 * than PCI.
+	 */

Can we perhaps force such sharing early temporarily even when the table is not filled up? This way we would get better test coverage of all of this.

That would be later disabled of course.

Rest looks ok to me.

-Andi

#5 lore raw

Re: [RFC][2.6.12.3] IRQ compression/sharing patch

Quoted 12 lines · Andi Kleen
Quoted 11 lines · James Cleverdon

diff -pruN 2.6.12.3/arch/i386/kernel/acpi/boot.c n12.3/arch/i386/kernel/acpi/boot.c ---
2.6.12.3/arch/i386/kernel/acpi/boot.c 2005-07-15 14:18:57.000000000
-0700 +++ n12.3/arch/i386/kernel/acpi/boot.c 2005-08-04
00:01:10.199710211 -0700 @@ -42,6 +42,7 @@
static inline void acpi_madt_oem_check(char *oem_id, char
*oem_table_id) { } extern void __init clustered_apic_check(void);
static inline int ioapic_setup_disabled(void) { return 0; }
+extern int gsi_irq_sharing(int gsi);
#include <asm/proto.h>

Quoted 8 lines

Review static inline int ioapic_setup_disabled( · line 55

Quoted 6 lines

#else /* X86 */

(quoted diff) +1 -0
@@ -51,6 +52,9 @@ static inline int ioapic_setup_disabled(
 #include <mach_mpparse.h>
 #endif	/* CONFIG_X86_LOCAL_APIC */
 
+static inline int gsi_irq_sharing(int gsi) { return gsi; }

Why is this different for i386/x86-64? It shouldn't.

True. Have added code for i386. Unfortunately, I didn't see one file that is shared by both architectures and which is included when building with I/O APIC support. So, I duplicated the function into io_apic.c

Quoted 18 lines

As a unrelated note we really need to get rid of this whole ifdef block.

Review config HAVE_DEC_LOCK · line 286

Quoted 12 lines
(quoted diff) +4 -3
+++ n12.3/arch/x86_64/Kconfig	2005-08-03 21:31:07.487451167 -0700
@@ -280,13 +280,13 @@ config HAVE_DEC_LOCK
 	default y
 
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "16"

Don't change the default please.

Quoted 3 lines

Review config HAVE_DEC_LOCK · line 287

Quoted 1 line
(quoted diff) +1 -0
+static int next_irq = 16;

Won't this need a lock for hotplug later?

That's what I thought originally, but maybe not. We initialize all RTEs and assign IRQs+vectors fairly early in boot, plus store the results in arrays. Thereafter the functions just return the preallocated values.

Hmmm... Since the I/O APIC init comes after the other CPUs are brought online, and since I don't understand all that the MSI driver is trying to accomplish, it might be safer to use a spin lock anyway.

Quoted 15 lines

Review

Quoted 10 lines
(quoted diff) +6 -0
+
+ retry_vector:
+	vector = assign_irq_vector(gsi);
+
+	/*
+	 * Sharing vectors means sharing IRQs, so scan irq_vectors for

previous + * use of vector and if found, return that IRQ.
However, we never want + * to share legacy IRQs, which usually
have a different trigger mode + * than PCI.

(quoted diff) +1 -0
+	 */

Can we perhaps force such sharing early temporarily even when the table is not filled up? This way we would get better test coverage of all of this.

Quoted 1 line

That would be later disabled of course.

Suppose I added a static counter and pretended that every third non-legacy IRQ needed to be shared?

Quoted 3 lines

Rest looks ok to me.

-Andi

Sigh. Have to attach the file again. Sorry about that.

Signature · 3 lines
James Cleverdon
IBM LTC (xSeries Linux Solutions)
{jamesclv(Unix, preferred), cleverdj(Notes)} at us dot ibm dot comm
arch/i386/kernel/acpi/boot.c +1 -1
diff -pru 2.6.12.3/arch/i386/kernel/acpi/boot.c z12.3/arch/i386/kernel/acpi/boot.c
@@ -453,7 +453,7 @@ int acpi_gsi_to_irq(u32 gsi, unsigned in
  		*irq = IO_APIC_VECTOR(gsi);
 	else
 #endif
-		*irq = gsi;
+		*irq = gsi_irq_sharing(gsi);
 	return 0;
 }
 
arch/i386/kernel/io_apic.c +86 -9

139 lines not on this page · raw message

arch/i386/kernel/mpparse.c +1 -1
diff -pru 2.6.12.3/arch/i386/kernel/mpparse.c z12.3/arch/i386/kernel/mpparse.c
@@ -274,7 +274,7 @@ static void __init MP_intsrc_info (struc
 			m->mpc_irqtype, m->mpc_irqflag & 3,
 			(m->mpc_irqflag >> 2) & 3, m->mpc_srcbus,
 			m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq);
-	if (++mp_irq_entries == MAX_IRQ_SOURCES)
+	if (++mp_irq_entries >= MAX_IRQ_SOURCES)
 		panic("Max # of irq sources exceeded!!\n");
 }
 
arch/x86_64/kernel/io_apic.c +91 -11

170 lines not on this page · raw message

arch/x86_64/kernel/mpparse.c +1 -1
diff -pru 2.6.12.3/arch/x86_64/kernel/mpparse.c z12.3/arch/x86_64/kernel/mpparse.c
@@ -214,7 +214,7 @@ static void __init MP_intsrc_info (struc
 			m->mpc_irqtype, m->mpc_irqflag & 3,
 			(m->mpc_irqflag >> 2) & 3, m->mpc_srcbus,
 			m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq);
-	if (++mp_irq_entries == MAX_IRQ_SOURCES)
+	if (++mp_irq_entries >= MAX_IRQ_SOURCES)
 		panic("Max # of irq sources exceeded!!\n");
 }
 
include/asm-i386/apic.h +1 -0
diff -pru 2.6.12.3/include/asm-i386/apic.h z12.3/include/asm-i386/apic.h
@@ -108,6 +108,7 @@ extern void nmi_watchdog_tick (struct pt
 extern int APIC_init_uniprocessor (void);
 extern void disable_APIC_timer(void);
 extern void enable_APIC_timer(void);
+extern int gsi_irq_sharing(int gsi);
 
 extern void enable_NMI_through_LVT0 (void * dummy);
 
include/asm-i386/mach-generic/mach_mpspec.h +2 -1
diff -pru 2.6.12.3/include/asm-i386/mach-generic/mach_mpspec.h z12.3/include/asm-i386/mach-generic/mach_mpspec.h
@@ -1,7 +1,8 @@
 #ifndef __ASM_MACH_MPSPEC_H
 #define __ASM_MACH_MPSPEC_H
 
-#define MAX_IRQ_SOURCES 256
+/* Each PCI slot may be a combo card with its own bus.  4 IRQ pins per slot. */
+#define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4)
 
 /* Summit or generic (i.e. installer) kernels need lots of bus entries. */
 /* Maximum 256 PCI busses, plus 1 ISA bus in each of 4 cabinets. */
include/asm-x86_64/apic.h +1 -0
diff -pru 2.6.12.3/include/asm-x86_64/apic.h z12.3/include/asm-x86_64/apic.h
@@ -98,6 +98,7 @@ extern int APIC_init_uniprocessor (void)
 extern void disable_APIC_timer(void);
 extern void enable_APIC_timer(void);
 extern void clustered_apic_check(void);
+extern int gsi_irq_sharing(int gsi);
 
 extern void nmi_watchdog_default(void);
 extern int setup_nmi_watchdog(char *);
include/asm-x86_64/mpspec.h +2 -1
diff -pru 2.6.12.3/include/asm-x86_64/mpspec.h z12.3/include/asm-x86_64/mpspec.h
@@ -157,7 +157,8 @@ struct mpc_config_lintsrc
  */
 
 #define MAX_MP_BUSSES 256
-#define MAX_IRQ_SOURCES 256
+/* Each PCI slot may be a combo card with its own bus.  4 IRQ pins per slot. */
+#define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4)
 enum mp_bustype {
 	MP_BUS_ISA = 1,
 	MP_BUS_EISA,
#6 lore raw

Re: [RFC][2.6.12.3] Use vectors 0x21-0x2F

Quick and dirty patch to use the 16 vectors that appear to have fallen through the cracks. This also is useful for older CPUs that have serial APICS: P54C, P6, P2, P3, K5, K6, etc. They can benefit from the extra vector space to hash the IRQs.

I suppose I could have started at 0x20 instead of 0x21. The extra level probably would more than compensate for having to skip 0x80 on the initial allocations.

include/asm-i386/mach-default/irq_vectors.h +3 -7
diff -pru 2.6.12.3/include/asm-i386/mach-default/irq_vectors.h z12.3/include/asm-i386/mach-default/irq_vectors.h
@@ -31,10 +31,6 @@
 #define SYSCALL_VECTOR		0x80
 
 /*
- * Vectors 0x20-0x2f are used for ISA interrupts.
- */
-
-/*
  * Special IRQ vectors used by the SMP architecture, 0xf0-0xff
  *
  *  some of the following vectors are 'rare', they are merged
@@ -58,11 +54,11 @@
 #define LOCAL_TIMER_VECTOR	0xef
 
 /*
- * First APIC vector available to drivers: (vectors 0x30-0xee)
- * we start at 0x31 to spread out vectors evenly between priority
+ * First APIC vector available to drivers: (vectors 0x20-0xee)
+ * we start at 0x21 to spread out vectors evenly between priority
  * levels. (0x80 is the syscall vector)
  */
-#define FIRST_DEVICE_VECTOR	0x31
+#define FIRST_DEVICE_VECTOR	0x21
 #define FIRST_SYSTEM_VECTOR	0xef
 
 #define TIMER_IRQ 0
include/asm-x86_64/hw_irq.h +3 -7
diff -pru 2.6.12.3/include/asm-x86_64/hw_irq.h z12.3/include/asm-x86_64/hw_irq.h
@@ -36,10 +36,6 @@ struct hw_interrupt_type;
 
 
 /*
- * Vectors 0x20-0x2f are used for ISA interrupts.
- */
-
-/*
  * Special IRQ vectors used by the SMP architecture, 0xf0-0xff
  *
  *  some of the following vectors are 'rare', they are merged
@@ -67,11 +63,11 @@ struct hw_interrupt_type;
 #define LOCAL_TIMER_VECTOR	0xef
 
 /*
- * First APIC vector available to drivers: (vectors 0x30-0xee)
- * we start at 0x31 to spread out vectors evenly between priority
+ * First APIC vector available to drivers: (vectors 0x20-0xee)
+ * we start at 0x21 to spread out vectors evenly between priority
  * levels. (0x80 is the syscall vector)
  */
-#define FIRST_DEVICE_VECTOR	0x31
+#define FIRST_DEVICE_VECTOR	0x21
 #define FIRST_SYSTEM_VECTOR	0xef   /* duplicated in irq.h */
 
 
Signature · 3 lines
James Cleverdon
IBM LTC (xSeries Linux Solutions)
{jamesclv(Unix, preferred), cleverdj(Notes)} at us dot ibm dot comm
#7 lore raw

Re: [RFC][2.6.12.3] IRQ compression/sharing patch

In reply to James Cleverdon
Quoted 11 lines · James Cleverdon
Quoted 3 lines
Quoted 1 line

+static int next_irq = 16;

Won't this need a lock for hotplug later?

That's what I thought originally, but maybe not. We initialize all RTEs and assign IRQs+vectors fairly early in boot, plus store the results in arrays. Thereafter the functions just return the preallocated values.

Hmmm... Since the I/O APIC init comes after the other CPUs are brought online, and since I don't understand all that the MSI driver is trying to accomplish, it might be safer to use a spin lock anyway.

With respect to vector allocation, the MSI driver locks around assign_irq_vector, it doesn't look like next_irq is used in that path so you shouldn't need a lock if it's only used in single threaded init. This of course would change if IOAPICs were added after boot.

#8 lore raw

Re: [RFC][2.6.12.3] IRQ compression/sharing patch

In reply to James Cleverdon
Quoted 18 lines · James Cleverdon
Quoted 17 lines · Andi Kleen
Quoted 16 lines · James Cleverdon

diff -pruN 2.6.12.3/arch/i386/kernel/acpi/boot.c n12.3/arch/i386/kernel/acpi/boot.c ---
2.6.12.3/arch/i386/kernel/acpi/boot.c 2005-07-15 14:18:57.000000000
-0700 +++ n12.3/arch/i386/kernel/acpi/boot.c 2005-08-04
00:01:10.199710211 -0700 @@ -42,6 +42,7 @@
static inline void acpi_madt_oem_check(char *oem_id, char
*oem_table_id) { } extern void __init clustered_apic_check(void);
static inline int ioapic_setup_disabled(void) { return 0; }

include/asm-i386/apic.h +1 -0
+extern int gsi_irq_sharing(int gsi);
 #include <asm/proto.h>
 
 #else	/* X86 */
@@ -51,6 +52,9 @@ static inline int ioapic_setup_disabled(
 #include <mach_mpparse.h>
 #endif	/* CONFIG_X86_LOCAL_APIC */
 
Quoted 8 lines
Quoted 3 lines

Review static inline int ioapic_setup_disabled( · line 55

Quoted 1 line
(quoted diff) +1 -0
+static inline int gsi_irq_sharing(int gsi) { return gsi; }

Why is this different for i386/x86-64? It shouldn't.

True. Have added code for i386. Unfortunately, I didn't see one file that is shared by both architectures and which is included when building with I/O APIC support. So, I duplicated the function into io_apic.c

That needs to be cleaned up before merge. This code is already ugly and I don't want the cruft accumulating here.

Quoted 18 lines
Quoted 18 lines

As a unrelated note we really need to get rid of this whole ifdef block.

Review config HAVE_DEC_LOCK · line 286

Quoted 12 lines
(quoted diff) +4 -3
+++ n12.3/arch/x86_64/Kconfig	2005-08-03 21:31:07.487451167 -0700
@@ -280,13 +280,13 @@ config HAVE_DEC_LOCK
 	default y
 
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "16"

Don't change the default please.

Quoted 7 lines
Quoted 3 lines

Review config HAVE_DEC_LOCK · line 287

Quoted 1 line
(quoted diff) +1 -0
+static int next_irq = 16;

Won't this need a lock for hotplug later?

That's what I thought originally, but maybe not. We initialize all RTEs and assign IRQs+vectors fairly early in boot, plus store the results in arrays. Thereafter the functions just return the preallocated values.

I was thinking of IO-APIC hotplug here. IIRC the ia64 folks have it already and I'm sure someone will turn up with a patch for i386/x86-64 soon. For devices it should be ok, you're right.

Ok I guess they can change it in that patch then. Perhaps just add a comment.

Quoted 11 lines
Quoted 8 lines

Review

Quoted 2 lines

have a different trigger mode + * than PCI.

(quoted diff) +1 -0
+	 */

Can we perhaps force such sharing early temporarily even when the table is not filled up? This way we would get better test coverage of all of this.

That would be later disabled of course.

Suppose I added a static counter and pretended that every third non-legacy IRQ needed to be shared?

Can you drop into the sharing path unconditionally?

-Andi

#9 lore raw

Re: [RFC][2.6.12.3] IRQ compression/sharing patch

Quoted 23 lines · Andi Kleen
Quoted 22 lines · James Cleverdon
Quoted 20 lines · Andi Kleen
Quoted 13 lines · James Cleverdon

diff -pruN 2.6.12.3/arch/i386/kernel/acpi/boot.c n12.3/arch/i386/kernel/acpi/boot.c ---
2.6.12.3/arch/i386/kernel/acpi/boot.c 2005-07-15 14:18:57.000000000
-0700 +++ n12.3/arch/i386/kernel/acpi/boot.c 2005-08-04
00:01:10.199710211 -0700 @@ -42,6 +42,7 @@
static inline void acpi_madt_oem_check(char *oem_id, char
*oem_table_id) { } extern void __init clustered_apic_check(void);
static inline int ioapic_setup_disabled(void) { return 0; }

(quoted diff) +1 -0
+extern int gsi_irq_sharing(int gsi);
 #include <asm/proto.h>
 
 #else	/* X86 */

Review static inline int ioapic_setup_disabled( · line 55

Quoted 4 lines
(quoted diff) +1 -0
@@ -51,6 +52,9 @@ static inline int ioapic_setup_disabled(
 #include <mach_mpparse.h>
 #endif	/* CONFIG_X86_LOCAL_APIC */
 
+static inline int gsi_irq_sharing(int gsi) { return gsi; }

Why is this different for i386/x86-64? It shouldn't.

Quoted 7 lines
Quoted 4 lines

True. Have added code for i386. Unfortunately, I didn't see one file that is shared by both architectures and which is included when building with I/O APIC support. So, I duplicated the function into io_apic.c

That needs to be cleaned up before merge. This code is already ugly and I don't want the cruft accumulating here.

OK, I moved the function into a separate file that can be used by both architectures.

Quoted 26 lines
Quoted 18 lines
Quoted 18 lines

As a unrelated note we really need to get rid of this whole ifdef block.

Review config HAVE_DEC_LOCK · line 286

Quoted 12 lines
(quoted diff) +4 -3
+++ n12.3/arch/x86_64/Kconfig	2005-08-03 21:31:07.487451167 -0700
@@ -280,13 +280,13 @@ config HAVE_DEC_LOCK
 	default y
 
 config NR_CPUS
-	int "Maximum number of CPUs (2-256)"
-	range 2 256
+	int "Maximum number of CPUs (2-255)"
+	range 2 255
 	depends on SMP
-	default "8"
+	default "16"

Don't change the default please.

Quoted 7 lines
Quoted 3 lines

Review config HAVE_DEC_LOCK · line 287

Quoted 1 line
(quoted diff) +1 -0
+static int next_irq = 16;

Won't this need a lock for hotplug later?

That's what I thought originally, but maybe not. We initialize all RTEs and assign IRQs+vectors fairly early in boot, plus store the results in arrays. Thereafter the functions just return the preallocated values.

Quoted 6 lines

I was thinking of IO-APIC hotplug here. IIRC the ia64 folks have it already and I'm sure someone will turn up with a patch for i386/x86-64 soon. For devices it should be ok, you're right.

Ok I guess they can change it in that patch then. Perhaps just add a comment.

I've already got a spin lock there, so may as well keep it.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Review

Quoted 2 lines

have a different trigger mode + * than PCI.

(quoted diff) +1 -0
+	 */

Can we perhaps force such sharing early temporarily even when the table is not filled up? This way we would get better test coverage of all of this.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

That would be later disabled of course.

Suppose I added a static counter and pretended that every third non-legacy IRQ needed to be shared?

Can you drop into the sharing path unconditionally?

-Andi

If no vectors/IRQs are ever allocated, there is nothing to share. Added some simple minded forced sharing to gsi_irq_sharing. It forces 1 in 3 IRQs to be shared. That should exercise some of the code paths.

Patch attached. (Sorry.)

Signature · 3 lines
James Cleverdon
IBM LTC (xSeries Linux Solutions)
{jamesclv(Unix, preferred), cleverdj(Notes)} at us dot ibm dot comm
arch/i386/kernel/Makefile +1 -1
diff -pruN 2.6.12.3/arch/i386/kernel/Makefile z12.3/arch/i386/kernel/Makefile
@@ -7,7 +7,7 @@ extra-y := head.o init_task.o vmlinux.ld
 obj-y	:= process.o semaphore.o signal.o entry.o traps.o irq.o vm86.o \
 		ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_i386.o \
 		pci-dma.o i386_ksyms.o i387.o dmi_scan.o bootflag.o \
-		doublefault.o quirks.o
+		doublefault.o quirks.o gsi2irq.o
 
 obj-y				+= cpu/
 obj-y				+= timers/
arch/i386/kernel/acpi/boot.c +1 -1
diff -pruN 2.6.12.3/arch/i386/kernel/acpi/boot.c z12.3/arch/i386/kernel/acpi/boot.c
@@ -453,7 +453,7 @@ int acpi_gsi_to_irq(u32 gsi, unsigned in
  		*irq = IO_APIC_VECTOR(gsi);
 	else
 #endif
-		*irq = gsi;
+		*irq = gsi_irq_sharing(gsi);
 	return 0;
 }
 
arch/i386/kernel/gsi2irq.c +134 -0

134 lines not on this page · raw message

arch/i386/kernel/io_apic.c +18 -9
diff -pruN 2.6.12.3/arch/i386/kernel/io_apic.c z12.3/arch/i386/kernel/io_apic.c
@@ -62,7 +62,7 @@ int nr_ioapic_registers[MAX_IO_APICS];
  * Rough estimation of how many shared IRQs there are, can
  * be changed anytime.
  */
-#define MAX_PLUS_SHARED_IRQS NR_IRQS
+#define MAX_PLUS_SHARED_IRQS NR_IRQ_VECTORS
 #define PIN_MAP_SIZE (MAX_PLUS_SHARED_IRQS + NR_IRQS)
 
 /*
@@ -1071,6 +1071,7 @@ static int pin_2_irq(int idx, int apic, 
 			while (i < apic)
 				irq += nr_ioapic_registers[i++];
 			irq += pin;
+			irq = gsi_irq_sharing(irq);
 
 			/*
 			 * For MPS mode, so far only needed by ES7000 platform
@@ -1127,11 +1128,11 @@ static inline int IO_APIC_irq_trigger(in
 /* irq_vectors is indexed by the sum of all RTEs in all I/O APICs. */
 u8 irq_vector[NR_IRQ_VECTORS] = { FIRST_DEVICE_VECTOR , 0 };
 
-int assign_irq_vector(int irq)
+int __assign_irq_vector(int irq)
 {
 	static int current_vector = FIRST_DEVICE_VECTOR, offset = 0;
 
-	BUG_ON(irq >= NR_IRQ_VECTORS);
+	BUG_ON(irq != AUTO_ASSIGN && (unsigned)irq >= NR_IRQ_VECTORS);
 	if (irq != AUTO_ASSIGN && IO_APIC_VECTOR(irq) > 0)
 		return IO_APIC_VECTOR(irq);
 next:
@@ -1140,17 +1141,24 @@ next:
 		goto next;
 
 	if (current_vector >= FIRST_SYSTEM_VECTOR) {
-		offset++;
-		if (!(offset%8))
-			return -ENOSPC;
+		/* If we run out of vectors on large boxen, must share them. */
+		offset = (offset + 1) % 8;
 		current_vector = FIRST_DEVICE_VECTOR + offset;
 	}
 
-	vector_irq[current_vector] = irq;
+	return current_vector;
+}
+
+int assign_irq_vector(int irq)
+{
+	int vect;
+
+	vect = __assign_irq_vector(irq);
+	vector_irq[vect] = irq;
 	if (irq != AUTO_ASSIGN)
-		IO_APIC_VECTOR(irq) = current_vector;
+		IO_APIC_VECTOR(irq) = vect;
 
-	return current_vector;
+	return vect;
 }
 
 static struct hw_interrupt_type ioapic_level_type;
@@ -2514,6 +2522,7 @@ int io_apic_set_pci_routing (int ioapic,
 	entry.polarity = active_high_low;
 	entry.mask  = 1;
 
+	irq = gsi_irq_sharing(irq);
 	/*
 	 * IRQs < 16 are already in the irq_2_pin[] map
 	 */
arch/i386/kernel/mpparse.c +1 -1
diff -pruN 2.6.12.3/arch/i386/kernel/mpparse.c z12.3/arch/i386/kernel/mpparse.c
@@ -274,7 +274,7 @@ static void __init MP_intsrc_info (struc
 			m->mpc_irqtype, m->mpc_irqflag & 3,
 			(m->mpc_irqflag >> 2) & 3, m->mpc_srcbus,
 			m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq);
-	if (++mp_irq_entries == MAX_IRQ_SOURCES)
+	if (++mp_irq_entries >= MAX_IRQ_SOURCES)
 		panic("Max # of irq sources exceeded!!\n");
 }
 
arch/x86_64/kernel/Makefile +2 -1
diff -pruN 2.6.12.3/arch/x86_64/kernel/Makefile z12.3/arch/x86_64/kernel/Makefile
@@ -7,7 +7,7 @@ EXTRA_AFLAGS	:= -traditional
 obj-y	:= process.o semaphore.o signal.o entry.o traps.o irq.o \
 		ptrace.o time.o ioport.o ldt.o setup.o i8259.o sys_x86_64.o \
 		x8664_ksyms.o i387.o syscall.o vsyscall.o \
-		setup64.o bootflag.o e820.o reboot.o quirks.o
+		setup64.o bootflag.o e820.o reboot.o quirks.o gsi2irq.o
 
 obj-$(CONFIG_X86_MCE)         += mce.o
 obj-$(CONFIG_X86_MCE_INTEL)	+= mce_intel.o
@@ -44,3 +44,4 @@ swiotlb-$(CONFIG_SWIOTLB)      += ../../
 microcode-$(subst m,y,$(CONFIG_MICROCODE))  += ../../i386/kernel/microcode.o
 intel_cacheinfo-y		+= ../../i386/kernel/cpu/intel_cacheinfo.o
 quirks-y			+= ../../i386/kernel/quirks.o
+gsi2irq-y			+= ../../i386/kernel/gsi2irq.o
arch/x86_64/kernel/io_apic.c +23 -11
diff -pruN 2.6.12.3/arch/x86_64/kernel/io_apic.c z12.3/arch/x86_64/kernel/io_apic.c
@@ -56,7 +56,7 @@ int nr_ioapic_registers[MAX_IO_APICS];
  * Rough estimation of how many shared IRQs there are, can
  * be changed anytime.
  */
-#define MAX_PLUS_SHARED_IRQS NR_IRQS
+#define MAX_PLUS_SHARED_IRQS NR_IRQ_VECTORS
 #define PIN_MAP_SIZE (MAX_PLUS_SHARED_IRQS + NR_IRQS)
 
 /*
@@ -88,6 +88,7 @@ static void add_pin_to_irq(unsigned int 
 	static int first_free_entry = NR_IRQS;
 	struct irq_pin_list *entry = irq_2_pin + irq;
 
+	BUG_ON(irq >= NR_IRQS);
 	while (entry->next)
 		entry = irq_2_pin + entry->next;
 
@@ -95,7 +96,7 @@ static void add_pin_to_irq(unsigned int 
 		entry->next = first_free_entry;
 		entry = irq_2_pin + entry->next;
 		if (++first_free_entry >= PIN_MAP_SIZE)
-			panic("io_apic.c: whoops");
+			panic("io_apic.c: ran out of irq_2_pin entries!");
 	}
 	entry->apic = apic;
 	entry->pin = pin;
@@ -610,6 +611,7 @@ static int pin_2_irq(int idx, int apic, 
 			while (i < apic)
 				irq += nr_ioapic_registers[i++];
 			irq += pin;
+			irq = gsi_irq_sharing(irq);
 			break;
 		}
 		default:
@@ -619,6 +621,7 @@ static int pin_2_irq(int idx, int apic, 
 			break;
 		}
 	}
+	BUG_ON(irq >= NR_IRQS);
 
 	/*
 	 * PCI IRQ command line redirection. Yes, limits are hardcoded.
@@ -634,6 +637,7 @@ static int pin_2_irq(int idx, int apic, 
 			}
 		}
 	}
+	BUG_ON(irq >= NR_IRQS);
 	return irq;
 }
 
@@ -657,12 +661,12 @@ static inline int IO_APIC_irq_trigger(in
 /* irq_vectors is indexed by the sum of all RTEs in all I/O APICs. */
 u8 irq_vector[NR_IRQ_VECTORS] = { FIRST_DEVICE_VECTOR , 0 };
 
-int assign_irq_vector(int irq)
+int __assign_irq_vector(int irq)
 {
 	static int current_vector = FIRST_DEVICE_VECTOR, offset = 0;
 
-	BUG_ON(irq >= NR_IRQ_VECTORS);
-	if (IO_APIC_VECTOR(irq) > 0)
+	BUG_ON(irq != AUTO_ASSIGN && (unsigned)irq >= NR_IRQ_VECTORS);
+	if (irq != AUTO_ASSIGN && IO_APIC_VECTOR(irq) > 0)
 		return IO_APIC_VECTOR(irq);
 next:
 	current_vector += 8;
@@ -670,17 +674,24 @@ next:
 		goto next;
 
 	if (current_vector >= FIRST_SYSTEM_VECTOR) {
-		offset++;
-		if (!(offset%8))
-			return -ENOSPC;
+		/* If we run out of vectors on large boxen, must share them. */
+		offset = (offset + 1) % 8;
 		current_vector = FIRST_DEVICE_VECTOR + offset;
 	}
 
-	vector_irq[current_vector] = irq;
+	return current_vector;
+}
+
+int assign_irq_vector(int irq)
+{
+	int vect;
+
+	vect = __assign_irq_vector(irq);
+	vector_irq[vect] = irq;
 	if (irq != AUTO_ASSIGN)
-		IO_APIC_VECTOR(irq) = current_vector;
+		IO_APIC_VECTOR(irq) = vect;
 
-	return current_vector;
+	return vect;
 }
 
 extern void (*interrupt[NR_IRQS])(void);
@@ -1866,6 +1877,7 @@ int io_apic_set_pci_routing (int ioapic,
 	entry.polarity = active_high_low;
 	entry.mask = 1;					 /* Disabled (masked) */
 
+	irq = gsi_irq_sharing(irq);
 	/*
 	 * IRQs < 16 are already in the irq_2_pin[] map
 	 */
arch/x86_64/kernel/mpparse.c +1 -1
diff -pruN 2.6.12.3/arch/x86_64/kernel/mpparse.c z12.3/arch/x86_64/kernel/mpparse.c
@@ -214,7 +214,7 @@ static void __init MP_intsrc_info (struc
 			m->mpc_irqtype, m->mpc_irqflag & 3,
 			(m->mpc_irqflag >> 2) & 3, m->mpc_srcbus,
 			m->mpc_srcbusirq, m->mpc_dstapic, m->mpc_dstirq);
-	if (++mp_irq_entries == MAX_IRQ_SOURCES)
+	if (++mp_irq_entries >= MAX_IRQ_SOURCES)
 		panic("Max # of irq sources exceeded!!\n");
 }
 
include/asm-i386/apic.h +1 -0
diff -pruN 2.6.12.3/include/asm-i386/apic.h z12.3/include/asm-i386/apic.h
@@ -108,6 +108,7 @@ extern void nmi_watchdog_tick (struct pt
 extern int APIC_init_uniprocessor (void);
 extern void disable_APIC_timer(void);
 extern void enable_APIC_timer(void);
+extern int gsi_irq_sharing(int gsi);
 
 extern void enable_NMI_through_LVT0 (void * dummy);
 
include/asm-i386/hw_irq.h +1 -0
diff -pruN 2.6.12.3/include/asm-i386/hw_irq.h z12.3/include/asm-i386/hw_irq.h
@@ -28,6 +28,7 @@
 extern u8 irq_vector[NR_IRQ_VECTORS];
 #define IO_APIC_VECTOR(irq)	(irq_vector[irq])
 #define AUTO_ASSIGN		-1
+extern int vector_irq[NR_VECTORS];
 
 extern void (*interrupt[NR_IRQS])(void);
 
include/asm-i386/mach-generic/mach_mpspec.h +2 -1
diff -pruN 2.6.12.3/include/asm-i386/mach-generic/mach_mpspec.h z12.3/include/asm-i386/mach-generic/mach_mpspec.h
@@ -1,7 +1,8 @@
 #ifndef __ASM_MACH_MPSPEC_H
 #define __ASM_MACH_MPSPEC_H
 
-#define MAX_IRQ_SOURCES 256
+/* Each PCI slot may be a combo card with its own bus.  4 IRQ pins per slot. */
+#define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4)
 
 /* Summit or generic (i.e. installer) kernels need lots of bus entries. */
 /* Maximum 256 PCI busses, plus 1 ISA bus in each of 4 cabinets. */
include/asm-x86_64/apic.h +1 -0
diff -pruN 2.6.12.3/include/asm-x86_64/apic.h z12.3/include/asm-x86_64/apic.h
@@ -98,6 +98,7 @@ extern int APIC_init_uniprocessor (void)
 extern void disable_APIC_timer(void);
 extern void enable_APIC_timer(void);
 extern void clustered_apic_check(void);
+extern int gsi_irq_sharing(int gsi);
 
 extern void nmi_watchdog_default(void);
 extern int setup_nmi_watchdog(char *);
include/asm-x86_64/hw_irq.h +1 -0
diff -pruN 2.6.12.3/include/asm-x86_64/hw_irq.h z12.3/include/asm-x86_64/hw_irq.h
@@ -79,6 +75,7 @@ struct hw_interrupt_type;
 extern u8 irq_vector[NR_IRQ_VECTORS];
 #define IO_APIC_VECTOR(irq)	(irq_vector[irq])
 #define AUTO_ASSIGN		-1
+extern int vector_irq[NR_VECTORS];
 
 /*
  * Various low-level irq details needed by irq.c, process.c,
include/asm-x86_64/mpspec.h +2 -1
diff -pruN 2.6.12.3/include/asm-x86_64/mpspec.h z12.3/include/asm-x86_64/mpspec.h
@@ -157,7 +157,8 @@ struct mpc_config_lintsrc
  */
 
 #define MAX_MP_BUSSES 256
-#define MAX_IRQ_SOURCES 256
+/* Each PCI slot may be a combo card with its own bus.  4 IRQ pins per slot. */
+#define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4)
 enum mp_bustype {
 	MP_BUS_ISA = 1,
 	MP_BUS_EISA,
#10 lore raw

Re: [RFC][2.6.12.3] IRQ compression/sharing patch

Patch looks good to me now. Thanks.
-Andi