Skip to contentSkip to conversation

Disable SW irqbalance/irqaffinity for E7520/E7320/E7525

PATCH Andi Kleen 5 participants 8 messages Sep 24, 2004 – Oct 1, 2004 1 message not in the archive PCI

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

Andi Kleen

author
#1 lore raw
Quoted 2 lines · Suresh Siddha

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline with x86 mach-default

And breaks compilation with MSI on. Here's a hackish workaround that will probably fail with >64 CPUs.
Proper fix would be to fix MSI to deal with cpumasks properly

-Andi

drivers/pci/msi.c renamed from drivers/pci/msi.c-o +2 -1
diff -u linux/drivers/pci/msi.c-o linux/drivers/pci/msi.c
@@ -282,7 +282,8 @@
 	msi_address->lo_address.u.dest_mode = MSI_DEST_MODE;
 	msi_address->lo_address.u.redirection_hint = MSI_REDIRECTION_HINT_MODE;
 	msi_address->lo_address.u.dest_id = dest_id;
-	msi_address->lo_address.value |= (MSI_TARGET_CPU << MSI_TARGET_CPU_SHIFT);
+	/* FIXME: broken for >64 CPUs */
+	msi_address->lo_address.value |= (*cpus_addr(MSI_TARGET_CPU) << MSI_TARGET_CPU_SHIFT);
 }
 
 static int msi_free_vector(struct pci_dev* dev, int vector, int reassign);

diff -u linux/include/asm-x86_64/smp.h-o linux/include/asm-x86_64/smp.h

#2 lore raw
Quoted 2 lines · Suresh Siddha
Quoted 2 lines

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline with x86 mach-default

Quoted 3 lines · Andi Kleen

And breaks compilation with MSI on. Here's a hackish workaround that will probably fail with >64 CPUs.
Proper fix would be to fix MSI to deal with cpumasks properly

I'll sweep this shortly (I must have forgotten to push the results from last time), sorry for the delay.

-- wli

#3 lore raw

Re: [Patch 1/2] Disable SW irqbalance/irqaffinity for E7520/E7320/E7525 - change TARGET_CPUS on x86_64

In reply to Andi Kleen
Quoted 6 lines · Andi Kleen
Quoted 2 lines · Suresh Siddha

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline with x86 mach-default

And breaks compilation with MSI on.

Here is a new patch. Andrew Please apply.

thanks,
suresh
--

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline with x86 mach-default. Fix MSI_TARGET_CPU code which will break with this target_cpus change.

arch/x86_64/kernel/io_apic.c +29 -4
diff -Nru linux-2.6.9-rc3/arch/x86_64/kernel/io_apic.c linux/arch/x86_64/kernel/io_apic.c
@@ -735,7 +735,7 @@
 		entry.delivery_mode = dest_LowestPrio;
 		entry.dest_mode = INT_DELIVERY_MODE;
 		entry.mask = 0;				/* enable IRQ */
-		entry.dest.logical.logical_dest = TARGET_CPUS;
+		entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 
 		idx = find_irq_entry(apic,pin,mp_INT);
 		if (idx == -1) {
@@ -753,7 +753,7 @@
 		if (irq_trigger(idx)) {
 			entry.trigger = 1;
 			entry.mask = 1;
-			entry.dest.logical.logical_dest = TARGET_CPUS;
+			entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 		}
 
 		irq = pin_2_irq(idx, apic, pin);
@@ -803,7 +803,7 @@
 	 */
 	entry.dest_mode = INT_DELIVERY_MODE;
 	entry.mask = 0;					/* unmask IRQ now */
-	entry.dest.logical.logical_dest = TARGET_CPUS;
+	entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 	entry.delivery_mode = dest_LowestPrio;
 	entry.polarity = 0;
 	entry.trigger = 0;
@@ -2011,7 +2011,7 @@
 
 	entry.delivery_mode = dest_LowestPrio;
 	entry.dest_mode = INT_DELIVERY_MODE;
-	entry.dest.logical.logical_dest = TARGET_CPUS;
+	entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 	entry.trigger = edge_level;
 	entry.polarity = active_high_low;
 	entry.mask = 1;					 /* Disabled (masked) */
@@ -2069,3 +2069,28 @@
 	apic_write_around(APIC_ICR, cfg);
 }
 #endif
+
+
+/*
+ * This function currently is only a helper for the i386 smp boot process where 
+ * we need to reprogram the ioredtbls to cater for the cpus which have come online
+ * so mask in all cases should simply be TARGET_CPUS
+ */
+void __init setup_ioapic_dest(void)
+{
+	int pin, ioapic, irq, irq_entry;
+
+	if (skip_ioapic_setup == 1)
+		return;
+
+	for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
+		for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
+			irq_entry = find_irq_entry(ioapic, pin, mp_INT);
+			if (irq_entry == -1)
+				continue;
+			irq = pin_2_irq(irq_entry, ioapic, pin);
+			set_ioapic_affinity_irq(irq, TARGET_CPUS);
+		}
+
+	}
+}
arch/x86_64/kernel/smpboot.c +3 -0
diff -Nru linux-2.6.9-rc3/arch/x86_64/kernel/smpboot.c linux/arch/x86_64/kernel/smpboot.c
@@ -950,6 +950,9 @@
 
 void __init smp_cpus_done(unsigned int max_cpus)
 {
+#ifdef CONFIG_X86_IO_APIC
+	setup_ioapic_dest();
+#endif
 	zap_low_mappings();
 }
 
include/asm-x86_64/hw_irq.h +1 -0
diff -Nru linux-2.6.9-rc3/include/asm-x86_64/hw_irq.h linux/include/asm-x86_64/hw_irq.h
@@ -101,6 +101,7 @@
 extern void print_IO_APIC(void);
 extern int IO_APIC_get_PCI_irq_vector(int bus, int slot, int fn);
 extern void send_IPI(int dest, int vector);
+extern void setup_ioapic_dest(void);
 
 extern unsigned long io_apic_irqs;
 
include/asm-x86_64/msi.h +5 -1
diff -Nru linux-2.6.9-rc3/include/asm-x86_64/msi.h linux/include/asm-x86_64/msi.h
@@ -11,6 +11,10 @@
 #define LAST_DEVICE_VECTOR		232
 #define MSI_DEST_MODE			MSI_LOGICAL_MODE
 #define MSI_TARGET_CPU_SHIFT		12
-#define MSI_TARGET_CPU			TARGET_CPUS
+#ifdef CONFIG_SMP
+#define MSI_TARGET_CPU			logical_smp_processor_id()
+#else
+#define MSI_TARGET_CPU			1
+#endif
 
 #endif /* ASM_MSI_H */
include/asm-x86_64/smp.h +11 -1
diff -Nru linux-2.6.9-rc3/include/asm-x86_64/smp.h linux/include/asm-x86_64/smp.h
@@ -74,6 +74,12 @@
 	return GET_APIC_ID(*(unsigned int *)(APIC_BASE+APIC_ID));
 }
 
+static __inline int logical_smp_processor_id(void)
+{
+	/* we don't want to mark this access volatile - bad code generation */
+	return GET_APIC_LOGICAL_ID(*(unsigned long *)(APIC_BASE+APIC_LDR));
+}
+
 /*
  * Some lowlevel functions might want to know about
  * the real APIC ID <-> CPU # mapping.
@@ -110,9 +116,13 @@
 
 #endif
 #define INT_DELIVERY_MODE 1     /* logical delivery */
-#define TARGET_CPUS 1
 
 #ifndef ASSEMBLY
+#ifdef CONFIG_SMP
+#define TARGET_CPUS cpu_online_map
+#else
+#define TARGET_CPUS cpumask_of_cpu(0)
+#endif
 static inline unsigned int cpu_mask_to_apicid(cpumask_t cpumask)
 {
 	return cpus_addr(cpumask)[0];
#4 lore raw

Re: [Patch 1/2] Disable SW irqbalance/irqaffinity for E7520/E7320/E7525 - change TARGET_CPUS on x86_64

Quoted 4 lines · Suresh Siddha

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline
with x86 mach-default. Fix MSI_TARGET_CPU code which will break with this target_cpus change.

This gets rejects all over the place against the x86_64 clustered APIC mode patch.

Which has priority here?

From: James Cleverdon <[email protected]>

Forthcoming IBM boxes will be using Nocona and/or Opteron chips in clustered mode to get beyond 8 CPUs. In fact, there are plans to try for 128 CPUs when the Tulsa chip comes out. Thus, there are a fair number of signed vs. unsigned changes in the patch.

Thanks to the HPET timer and some HW changes, I've been able to remove the MPS/ACPI string comparisons from the detection code. Instead, it scans bios_cpu_apicid and uses simple heuristics to select the correct IRQ delivery mode. No need for a config option. Hurrah!

Likewise, I've been able to avoid the preprocessor tricks that the i386 sub-arch needed to build with one or more sub-arches.

---

 25-akpm/arch/i386/kernel/acpi/boot.c         |    2 
 25-akpm/arch/x86_64/kernel/Makefile          |    3 
 25-akpm/arch/x86_64/kernel/apic.c            |   22 ----
 25-akpm/arch/x86_64/kernel/genapic.c         |   80 ++++++++++++++
 25-akpm/arch/x86_64/kernel/genapic_cluster.c |  146 +++++++++++++++++++++++++++
 25-akpm/arch/x86_64/kernel/genapic_flat.c    |  119 ++++++++++++++++++++++
 25-akpm/arch/x86_64/kernel/io_apic.c         |   60 ++---------
 25-akpm/arch/x86_64/kernel/smp.c             |  100 ------------------
 25-akpm/arch/x86_64/kernel/smpboot.c         |    9 -
 25-akpm/include/asm-x86_64/apic.h            |    1 
 25-akpm/include/asm-x86_64/apicdef.h         |   33 ++++--
 25-akpm/include/asm-x86_64/genapic.h         |   34 ++++++
 25-akpm/include/asm-x86_64/ipi.h             |  113 ++++++++++++++++++++
 25-akpm/include/asm-x86_64/irq.h             |    2 
 25-akpm/include/asm-x86_64/mach_apic.h       |   28 +++++
 25-akpm/include/asm-x86_64/mpspec.h          |    4 
 25-akpm/include/asm-x86_64/msi.h             |    1 
 25-akpm/include/asm-x86_64/smp.h             |   41 +++----
 18 files changed, 588 insertions(+), 210 deletions(-)
arch/i386/kernel/acpi/boot.c renamed from arch/i386/kernel/acpi/boot.c~x86-64-clustered-apic-support +1 -1
diff -puN arch/i386/kernel/acpi/boot.c~x86-64-clustered-apic-support arch/i386/kernel/acpi/boot.c
@@ -40,7 +40,7 @@
 #ifdef	CONFIG_X86_64
 
 static inline void  acpi_madt_oem_check(char *oem_id, char *oem_table_id) { }
-static inline void clustered_apic_check(void) { }
+extern void __init clustered_apic_check(void);
 static inline int ioapic_setup_disabled(void) { return 0; }
 #include <asm/proto.h>
 
arch/x86_64/kernel/apic.c renamed from arch/x86_64/kernel/apic.c~x86-64-clustered-apic-support +3 -19
diff -puN arch/x86_64/kernel/apic.c~x86-64-clustered-apic-support arch/x86_64/kernel/apic.c
@@ -32,6 +32,7 @@
 #include <asm/mtrr.h>
 #include <asm/mpspec.h>
 #include <asm/pgalloc.h>
+#include <asm/mach_apic.h>
 
 int apic_verbosity;
 
@@ -330,8 +331,7 @@ void __init setup_local_APIC (void)
 	 * Double-check whether this APIC is really registered.
 	 * This is meaningless in clustered apic mode, so we skip it.
 	 */
-	if (!clustered_apic_mode &&
-		!physid_isset(GET_APIC_ID(apic_read(APIC_ID)), phys_cpu_present_map))
+	if (!apic_id_registered())
 		BUG();
 
 	/*
@@ -339,23 +339,7 @@ void __init setup_local_APIC (void)
 	 * an APIC.  See e.g. "AP-388 82489DX User's Manual" (Intel
 	 * document number 292116).  So here it goes...
 	 */
-
-	if (!clustered_apic_mode) {
-		/*
-		 * In clustered apic mode, the firmware does this for us 
-		 * Put the APIC into flat delivery mode.
-		 * Must be "all ones" explicitly for 82489DX.
-		 */
-		apic_write_around(APIC_DFR, 0xffffffff);
-
-		/*
-		 * Set up the logical destination ID.
-		 */
-		value = apic_read(APIC_LDR);
-		value &= ~APIC_LDR_MASK;
-		value |= (1<<(smp_processor_id()+24));
-		apic_write_around(APIC_LDR, value);
-	}
+	init_apic_ldr();
 
 	/*
 	 * Set Task Priority to 'accept all'. We never change this
arch/x86_64/kernel/genapic.c new file +80 -0

80 lines not on this page · raw message

arch/x86_64/kernel/genapic_cluster.c new file +146 -0

146 lines not on this page · raw message

arch/x86_64/kernel/genapic_flat.c new file +119 -0

119 lines not on this page · raw message

arch/x86_64/kernel/io_apic.c renamed from arch/x86_64/kernel/io_apic.c~x86-64-clustered-apic-support +11 -49

114 lines not on this page · raw message

arch/x86_64/kernel/Makefile renamed from arch/x86_64/kernel/Makefile~x86-64-clustered-apic-support +2 -1
diff -puN arch/x86_64/kernel/Makefile~x86-64-clustered-apic-support arch/x86_64/kernel/Makefile
@@ -17,7 +17,8 @@ obj-$(CONFIG_MICROCODE)		+= microcode.o
 obj-$(CONFIG_X86_CPUID)		+= cpuid.o
 obj-$(CONFIG_SMP)		+= smp.o smpboot.o trampoline.o
 obj-$(CONFIG_X86_LOCAL_APIC)	+= apic.o  nmi.o
-obj-$(CONFIG_X86_IO_APIC)	+= io_apic.o mpparse.o
+obj-$(CONFIG_X86_IO_APIC)	+= io_apic.o mpparse.o \
+		genapic.o genapic_cluster.o genapic_flat.o
 obj-$(CONFIG_KEXEC)		+= machine_kexec.o relocate_kernel.o
 obj-$(CONFIG_PM)		+= suspend.o
 obj-$(CONFIG_SOFTWARE_SUSPEND)	+= suspend_asm.o
arch/x86_64/kernel/smpboot.c renamed from arch/x86_64/kernel/smpboot.c~x86-64-clustered-apic-support +4 -5
diff -puN arch/x86_64/kernel/smpboot.c~x86-64-clustered-apic-support arch/x86_64/kernel/smpboot.c
@@ -56,15 +56,12 @@
 
 /* Number of siblings per CPU package */
 int smp_num_siblings = 1;
-char phys_proc_id[NR_CPUS]; /* Package ID of each logical CPU */
+/* Package ID of each logical CPU */
+u8 phys_proc_id[NR_CPUS] = { [0 ... NR_CPUS-1] = BAD_APICID };
 
 /* Bitmask of currently online CPUs */
 cpumask_t cpu_online_map;
 
-/* which logical CPU number maps to which CPU (physical APIC ID) */
-volatile char x86_cpu_to_apicid[NR_CPUS];
-EXPORT_SYMBOL(x86_cpu_to_apicid);
-
 static cpumask_t cpu_callin_map;
 cpumask_t cpu_callout_map;
 static cpumask_t smp_commenced_mask;
@@ -658,6 +655,8 @@ static void __init do_boot_cpu (int apic
 		cpu_clear(cpu, cpu_callout_map); /* was set here (do_boot_cpu()) */
 		clear_bit(cpu, &cpu_initialized); /* was set by cpu_init() */
 		cpucount--;
+		x86_cpu_to_apicid[cpu] = BAD_APICID;
+		x86_cpu_to_log_apicid[cpu] = BAD_APICID;
 	}
 }
 
arch/x86_64/kernel/smp.c renamed from arch/x86_64/kernel/smp.c~x86-64-clustered-apic-support +1 -99

106 lines not on this page · raw message

include/asm-x86_64/apicdef.h renamed from include/asm-x86_64/apicdef.h~x86-64-clustered-apic-support +23 -10
diff -puN include/asm-x86_64/apicdef.h~x86-64-clustered-apic-support include/asm-x86_64/apicdef.h
@@ -11,26 +11,26 @@
 #define		APIC_DEFAULT_PHYS_BASE	0xfee00000
  
 #define		APIC_ID		0x20
-#define			APIC_ID_MASK		(0x0F<<24)
-#define			GET_APIC_ID(x)		(((x)>>24)&0x0F)
+#define			APIC_ID_MASK		(0xFFu<<24)
+#define			GET_APIC_ID(x)		(((x)>>24)&0xFFu)
 #define		APIC_LVR	0x30
 #define			APIC_LVR_MASK		0xFF00FF
-#define			GET_APIC_VERSION(x)	((x)&0xFF)
-#define			GET_APIC_MAXLVT(x)	(((x)>>16)&0xFF)
-#define			APIC_INTEGRATED(x)	((x)&0xF0)
+#define			GET_APIC_VERSION(x)	((x)&0xFFu)
+#define			GET_APIC_MAXLVT(x)	(((x)>>16)&0xFFu)
+#define			APIC_INTEGRATED(x)	((x)&0xF0u)
 #define		APIC_TASKPRI	0x80
-#define			APIC_TPRI_MASK		0xFF
+#define			APIC_TPRI_MASK		0xFFu
 #define		APIC_ARBPRI	0x90
-#define			APIC_ARBPRI_MASK	0xFF
+#define			APIC_ARBPRI_MASK	0xFFu
 #define		APIC_PROCPRI	0xA0
 #define		APIC_EOI	0xB0
 #define			APIC_EIO_ACK		0x0		/* Write this to the EOI register */
 #define		APIC_RRR	0xC0
 #define		APIC_LDR	0xD0
-#define			APIC_LDR_MASK		(0xFF<<24)
-#define			GET_APIC_LOGICAL_ID(x)	(((x)>>24)&0xFF)
+#define			APIC_LDR_MASK		(0xFFu<<24)
+#define			GET_APIC_LOGICAL_ID(x)	(((x)>>24)&0xFFu)
 #define			SET_APIC_LOGICAL_ID(x)	(((x)<<24))
-#define			APIC_ALL_CPUS		0xFF
+#define			APIC_ALL_CPUS		0xFFu
 #define		APIC_DFR	0xE0
 #define			APIC_DFR_CLUSTER		0x0FFFFFFFul
 #define			APIC_DFR_FLAT			0xFFFFFFFFul
@@ -60,6 +60,7 @@
 #define			APIC_INT_ASSERT		0x04000
 #define			APIC_ICR_BUSY		0x01000
 #define			APIC_DEST_LOGICAL	0x00800
+#define			APIC_DEST_PHYSICAL	0x00000
 #define			APIC_DM_FIXED		0x00000
 #define			APIC_DM_LOWEST		0x00100
 #define			APIC_DM_SMI		0x00200
@@ -114,6 +115,18 @@
 #define MAX_IO_APICS 32
 
 /*
+ * All x86-64 systems are xAPIC compatible.
+ * In the following, "apicid" is a physical APIC ID.
+ */
+#define XAPIC_DEST_CPUS_SHIFT	4
+#define XAPIC_DEST_CPUS_MASK	((1u << XAPIC_DEST_CPUS_SHIFT) - 1)
+#define XAPIC_DEST_CLUSTER_MASK	(XAPIC_DEST_CPUS_MASK << XAPIC_DEST_CPUS_SHIFT)
+#define APIC_CLUSTER(apicid)	((apicid) & XAPIC_DEST_CLUSTER_MASK)
+#define APIC_CLUSTERID(apicid)	(APIC_CLUSTER(apicid) >> XAPIC_DEST_CPUS_SHIFT)
+#define APIC_CPUID(apicid)	((apicid) & XAPIC_DEST_CPUS_MASK)
+#define NUM_APIC_CLUSTERS	((BAD_APICID + 1) >> XAPIC_DEST_CPUS_SHIFT)
+
+/*
  * the local APIC register structure, memory mapped. Not terribly well
  * tested, but we might eventually use this one in the future - the
  * problem why we cannot use it right now is the P5 APIC, it has an
include/asm-x86_64/apic.h renamed from include/asm-x86_64/apic.h~x86-64-clustered-apic-support +0 -1
diff -puN include/asm-x86_64/apic.h~x86-64-clustered-apic-support include/asm-x86_64/apic.h
@@ -111,7 +111,6 @@ extern unsigned int nmi_watchdog;
 
 #endif /* CONFIG_X86_LOCAL_APIC */
 
-#define clustered_apic_mode 0
 #define esr_disable 0
 extern unsigned boot_cpu_id;
 
include/asm-x86_64/genapic.h new file +34 -0
diff -puN /dev/null include/asm-x86_64/genapic.h
@@ -0,0 +1,34 @@
+#ifndef _ASM_GENAPIC_H
+#define _ASM_GENAPIC_H 1
+
+/*
+ * Copyright 2004 James Cleverdon, IBM.
+ * Subject to the GNU Public License, v.2
+ *
+ * Generic APIC sub-arch data struct.
+ *
+ * Hacked for x86-64 by James Cleverdon from i386 architecture code by
+ * Martin Bligh, Andi Kleen, James Bottomley, John Stultz, and
+ * James Cleverdon.
+ */
+
+struct genapic {
+	char *name;
+	u32 int_delivery_mode;
+	u32 int_dest_mode;
+	u32 int_delivery_dest;	/* for quick IPIs */
+	int (*apic_id_registered)(void);
+	u8 (*target_cpus)(void);
+	void (*init_apic_ldr)(void);
+	/* ipi */
+	void (*send_IPI_mask)(cpumask_t mask, int vector);
+	void (*send_IPI_allbutself)(int vector);
+	void (*send_IPI_all)(int vector);
+	/* */
+	unsigned int (*cpu_mask_to_apicid)(cpumask_t cpumask);
+};
+
+
+extern struct genapic *genapic;
+
+#endif
include/asm-x86_64/ipi.h new file +113 -0

113 lines not on this page · raw message

include/asm-x86_64/irq.h renamed from include/asm-x86_64/irq.h~x86-64-clustered-apic-support +1 -1
diff -puN include/asm-x86_64/irq.h~x86-64-clustered-apic-support include/asm-x86_64/irq.h
@@ -36,7 +36,7 @@
 #define NR_IRQ_VECTORS NR_IRQS
 #else
 #define NR_IRQS 224
-#define NR_IRQ_VECTORS NR_IRQS
+#define NR_IRQ_VECTORS 1024
 #endif
 
 static __inline__ int irq_canonicalize(int irq)
include/asm-x86_64/mach_apic.h new file +28 -0
diff -puN /dev/null include/asm-x86_64/mach_apic.h
@@ -0,0 +1,28 @@
+#ifndef __ASM_MACH_APIC_H
+#define __ASM_MACH_APIC_H
+
+/*
+ * Copyright 2004 James Cleverdon, IBM.
+ * Subject to the GNU Public License, v.2
+ *
+ * Generic APIC sub-arch defines.
+ *
+ * Hacked for x86-64 by James Cleverdon from i386 architecture code by
+ * Martin Bligh, Andi Kleen, James Bottomley, John Stultz, and
+ * James Cleverdon.
+ */
+
+#include <asm/genapic.h>
+
+#define INT_DELIVERY_MODE (genapic->int_delivery_mode)
+#define INT_DEST_MODE (genapic->int_dest_mode)
+#define INT_DELIVERY_DEST (genapic->int_delivery_dest)
+#define TARGET_CPUS	  (genapic->target_cpus())
+#define apic_id_registered (genapic->apic_id_registered)
+#define init_apic_ldr (genapic->init_apic_ldr)
+#define send_IPI_mask (genapic->send_IPI_mask)
+#define send_IPI_allbutself (genapic->send_IPI_allbutself)
+#define send_IPI_all (genapic->send_IPI_all)
+#define cpu_mask_to_apicid (genapic->cpu_mask_to_apicid)
+
+#endif /* __ASM_MACH_APIC_H */
include/asm-x86_64/mpspec.h renamed from include/asm-x86_64/mpspec.h~x86-64-clustered-apic-support +2 -2
diff -puN include/asm-x86_64/mpspec.h~x86-64-clustered-apic-support include/asm-x86_64/mpspec.h
@@ -14,9 +14,9 @@
 #define SMP_MAGIC_IDENT	(('_'<<24)|('P'<<16)|('M'<<8)|'_')
 
 /*
- * a maximum of 16 APICs with the current APIC ID architecture.
+ * A maximum of 255 APICs with the current APIC ID architecture.
  */
-#define MAX_APICS 16
+#define MAX_APICS 128
 
 struct intel_mp_floating
 {
include/asm-x86_64/msi.h renamed from include/asm-x86_64/msi.h~x86-64-clustered-apic-support +1 -0
diff -puN include/asm-x86_64/msi.h~x86-64-clustered-apic-support include/asm-x86_64/msi.h
@@ -7,6 +7,7 @@
 #define ASM_MSI_H
 
 #include <asm/desc.h>
+#include <asm/mach_apic.h>
 
 #define LAST_DEVICE_VECTOR		232
 #define MSI_DEST_MODE			MSI_LOGICAL_MODE
include/asm-x86_64/smp.h renamed from include/asm-x86_64/smp.h~x86-64-clustered-apic-support +19 -22

70 lines not on this page · raw message

_

author
#5 lore raw

Re: [Patch 1/2] Disable SW irqbalance/irqaffinity for E7520/E7320/E7525 - change TARGET_CPUS on x86_64

Quoted 10 lines · Andrew Morton
Quoted 4 lines · Suresh Siddha

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline
with x86 mach-default. Fix MSI_TARGET_CPU code which will break with this target_cpus change.

This gets rejects all over the place against the x86_64 clustered APIC mode patch.

Which has priority here?

Definitely the MSI_TARGET_CPUS thingy.

The Clustered APIC patch is far off pie in the sky for some future unreleased hardware. MSI workaround fixes basic compilation and the original patch from Suresh fixes shipping Intel chipsets.

-Andi

#6 lore raw

Re: [Patch 1/2] Disable SW irqbalance/irqaffinity for E7520/E7320/E7525 - change TARGET_CPUS on x86_64

Quoted 11 lines · Andi Kleen
Quoted 9 lines · Andrew Morton
Quoted 3 lines · Suresh Siddha

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline with x86 mach-default. Fix MSI_TARGET_CPU code which will break with this target_cpus change.

This gets rejects all over the place against the x86_64 clustered APIC mode patch.

Which has priority here?

Quoted 7 lines

Definitely the MSI_TARGET_CPUS thingy.

The Clustered APIC patch is far off pie in the sky for some future unreleased hardware. MSI workaround fixes basic compilation and the original patch from Suresh fixes shipping Intel chipsets.

-Andi

Excuse me, but since when is February "far off pie in the sky for some future unreleased hardware"?

Zeus boxes are going out the door 1Q2005. The question is, will v2.6 work on them or not?

Signature · 3 lines
James Cleverdon
IBM LTC (xSeries Linux Solutions)
{jamesclv(Unix, preferred), cleverdj(Notes)} at us dot ibm dot comm
#7 lore raw

Re: [Patch 1/2] Disable SW irqbalance/irqaffinity for E7520/E7320/E7525 - change TARGET_CPUS on x86_64

Quoted 3 lines · James Cleverdon

Zeus boxes are going out the door 1Q2005. The question is, will v2.6 work on them or not?

We'll get it into 2.6.10. Please rework your patch against the below, or against next -mm.

From: Suresh Siddha <[email protected]>

Set TARGET_CPUS on x86_64 to cpu_online_map. This brings the code inline with x86 mach-default. Fix MSI_TARGET_CPU code which will break with this target_cpus change.

---

 25-akpm/arch/x86_64/kernel/io_apic.c |   33 +++++++++++++++++++++++++++++----
 25-akpm/arch/x86_64/kernel/smpboot.c |    3 +++
 25-akpm/include/asm-x86_64/hw_irq.h  |    1 +
 25-akpm/include/asm-x86_64/msi.h     |    6 +++++-
 25-akpm/include/asm-x86_64/smp.h     |   12 +++++++++++-
 5 files changed, 49 insertions(+), 6 deletions(-)
arch/x86_64/kernel/io_apic.c renamed from arch/x86_64/kernel/io_apic.c~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 +29 -4
diff -puN arch/x86_64/kernel/io_apic.c~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 arch/x86_64/kernel/io_apic.c
@@ -731,7 +731,7 @@ void __init setup_IO_APIC_irqs(void)
 		entry.delivery_mode = dest_LowestPrio;
 		entry.dest_mode = INT_DELIVERY_MODE;
 		entry.mask = 0;				/* enable IRQ */
-		entry.dest.logical.logical_dest = TARGET_CPUS;
+		entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 
 		idx = find_irq_entry(apic,pin,mp_INT);
 		if (idx == -1) {
@@ -749,7 +749,7 @@ void __init setup_IO_APIC_irqs(void)
 		if (irq_trigger(idx)) {
 			entry.trigger = 1;
 			entry.mask = 1;
-			entry.dest.logical.logical_dest = TARGET_CPUS;
+			entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 		}
 
 		irq = pin_2_irq(idx, apic, pin);
@@ -799,7 +799,7 @@ void __init setup_ExtINT_IRQ0_pin(unsign
 	 */
 	entry.dest_mode = INT_DELIVERY_MODE;
 	entry.mask = 0;					/* unmask IRQ now */
-	entry.dest.logical.logical_dest = TARGET_CPUS;
+	entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 	entry.delivery_mode = dest_LowestPrio;
 	entry.polarity = 0;
 	entry.trigger = 0;
@@ -2039,7 +2039,7 @@ int io_apic_set_pci_routing (int ioapic,
 
 	entry.delivery_mode = dest_LowestPrio;
 	entry.dest_mode = INT_DELIVERY_MODE;
-	entry.dest.logical.logical_dest = TARGET_CPUS;
+	entry.dest.logical.logical_dest = cpu_mask_to_apicid(TARGET_CPUS);
 	entry.trigger = edge_level;
 	entry.polarity = active_high_low;
 	entry.mask = 1;					 /* Disabled (masked) */
@@ -2089,3 +2089,28 @@ void send_IPI_self(int vector)
 	apic_write_around(APIC_ICR, cfg);
 }
 #endif
+
+
+/*
+ * This function currently is only a helper for the i386 smp boot process where
+ * we need to reprogram the ioredtbls to cater for the cpus which have come online
+ * so mask in all cases should simply be TARGET_CPUS
+ */
+void __init setup_ioapic_dest(void)
+{
+	int pin, ioapic, irq, irq_entry;
+
+	if (skip_ioapic_setup == 1)
+		return;
+
+	for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
+		for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
+			irq_entry = find_irq_entry(ioapic, pin, mp_INT);
+			if (irq_entry == -1)
+				continue;
+			irq = pin_2_irq(irq_entry, ioapic, pin);
+			set_ioapic_affinity_irq(irq, TARGET_CPUS);
+		}
+
+	}
+}
arch/x86_64/kernel/smpboot.c renamed from arch/x86_64/kernel/smpboot.c~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 +3 -0
diff -puN arch/x86_64/kernel/smpboot.c~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 arch/x86_64/kernel/smpboot.c
@@ -950,6 +950,9 @@ int __devinit __cpu_up(unsigned int cpu)
 
 void __init smp_cpus_done(unsigned int max_cpus)
 {
+#ifdef CONFIG_X86_IO_APIC
+	setup_ioapic_dest();
+#endif
 	zap_low_mappings();
 }
 
include/asm-x86_64/hw_irq.h renamed from include/asm-x86_64/hw_irq.h~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 +1 -0
diff -puN include/asm-x86_64/hw_irq.h~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 include/asm-x86_64/hw_irq.h
@@ -102,6 +102,7 @@ extern void disable_IO_APIC(void);
 extern void print_IO_APIC(void);
 extern int IO_APIC_get_PCI_irq_vector(int bus, int slot, int fn);
 extern void send_IPI(int dest, int vector);
+extern void setup_ioapic_dest(void);
 
 extern unsigned long io_apic_irqs;
 
include/asm-x86_64/msi.h renamed from include/asm-x86_64/msi.h~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 +5 -1
diff -puN include/asm-x86_64/msi.h~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 include/asm-x86_64/msi.h
@@ -11,6 +11,10 @@
 #define LAST_DEVICE_VECTOR		232
 #define MSI_DEST_MODE			MSI_LOGICAL_MODE
 #define MSI_TARGET_CPU_SHIFT		12
-#define MSI_TARGET_CPU			TARGET_CPUS
+#ifdef CONFIG_SMP
+#define MSI_TARGET_CPU			logical_smp_processor_id()
+#else
+#define MSI_TARGET_CPU			1
+#endif
 
 #endif /* ASM_MSI_H */
include/asm-x86_64/smp.h renamed from include/asm-x86_64/smp.h~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 +11 -1
diff -puN include/asm-x86_64/smp.h~disable-sw-irqbalance-irqaffinity-for-e7520-e7320-e7525-change-target_cpus-on-x86_64 include/asm-x86_64/smp.h
@@ -74,6 +74,12 @@ extern __inline int hard_smp_processor_i
 	return GET_APIC_ID(*(unsigned int *)(APIC_BASE+APIC_ID));
 }
 
+static __inline int logical_smp_processor_id(void)
+{
+	/* we don't want to mark this access volatile - bad code generation */
+	return GET_APIC_LOGICAL_ID(*(unsigned long *)(APIC_BASE+APIC_LDR));
+}
+
 /*
  * Some lowlevel functions might want to know about
  * the real APIC ID <-> CPU # mapping.
@@ -110,9 +116,13 @@ static inline int cpu_present_to_apicid(
 
 #endif
 #define INT_DELIVERY_MODE 1     /* logical delivery */
-#define TARGET_CPUS 1
 
 #ifndef ASSEMBLY
+#ifdef CONFIG_SMP
+#define TARGET_CPUS cpu_online_map
+#else
+#define TARGET_CPUS cpumask_of_cpu(0)
+#endif
 static inline unsigned int cpu_mask_to_apicid(cpumask_t cpumask)
 {
 	return cpus_addr(cpumask)[0];

_

author
#8 lore raw

Re: [Patch 1/2] Disable SW irqbalance/irqaffinity for E7520/E7320/E7525 - change TARGET_CPUS on x86_64

In reply to James Cleverdon
Quoted 2 lines

Excuse me, but since when is February "far off pie in the sky for some future unreleased hardware"?

Compared to currently shipping Intel chipsets it definitely is.

Quoted 2 lines

Zeus boxes are going out the door 1Q2005. The question is, will v2.6 work on them or not?

It will at some point, after the intrusive patches needed to support their pecularities have been well enough tested.

-Andi