Skip to contentSkip to conversation

x86: Reduce memory usage for large count NR_CPUs fixup V2

PATCH New [email protected] 1 participant 5 messages Jan 21, 2008

Subsystems (3)
Memory management x86 x86 MM
lore
Share & tools
lore

[email protected]

author
#0 lore raw

cover x86: Reduce memory usage for large count NR_CPUs fixup V2

Fixup change NR_CPUS patchset by rebasing on 2.6.24-rc8-mm1 (from 2.6.24-rc6-mm1) and adding changes suggested by reviews.

Based on 2.6.24-rc8-mm1

Note there are two versions of this patchset:
- 2.6.24-rc8-mm1
- 2.6.24-rc8-mm1 + latest (08/1/21) git-x86

Signed-off-by Mike Travis <[email protected]>

---
Fixup-V2:
- pulled the SMP_MAX patch as it's not strictly needed and some more work on local cpumask_t variables needs to be done before NR_CPUS is allowed to increase.

- changes to X86_32 have been removed (except for build errors)

Signature · 5 lines
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
author
#1 lore raw

PATCH 1/4 x86: Change size of node ids from u8 to s16 fixup V2

Change the size of node ids for X86_64 from u8 to s16 to accomodate more than 32k nodes and allow for NUMA_NO_NODE (-1) to be sign extended to int.

Based on 2.6.24-rc8-mm1

Cc David Rientjes <[email protected]>
Cc Yinghai Lu <[email protected]>
Cc Eric Dumazet <[email protected]>
Signed-off-by Mike Travis <[email protected]>
Reviewed-by Christoph Lameter <[email protected]>

---
fixup-V2:

- Fixed populate_memnodemap as suggested by Eric.
- Change to using s16 for static node id arrays and int for node id's in per_cpu variables and __initdata arrays as suggested by David and Yinghai.
- NUMA_NO_NODE is now (-1)

fixup:

- Size of memnode.embedded_map needs to be changed to accomodate 16-bit node ids as suggested by Eric.

V2->V3:
- changed memnode.embedded_map from [64-16] to [64-8] (and size comment to 128 bytes)

V1->V2:
- changed pxm_to_node_map to u16
- changed memnode map entries to u16

---
 arch/x86/Kconfig            |    1 +
 arch/x86/mm/numa_64.c       |   14 +++++++-------
 include/asm-x86/mmzone_64.h |    6 +++---
 include/asm-x86/numa_64.h   |    2 +-
 include/asm-x86/topology.h  |    8 ++++----
 5 files changed, 16 insertions(+), 15 deletions(-)
arch/x86/Kconfig +1 -0
@@ -882,6 +882,7 @@ config NUMA_EMU
 
 config NODES_SHIFT
 	int
+	range 1 15  if X86_64
 	default "6" if X86_64
 	default "4" if X86_NUMAQ
 	default "3"
arch/x86/mm/numa_64.c +7 -7
@@ -31,16 +31,16 @@ bootmem_data_t plat_node_bdata[MAX_NUMNO
 
 struct memnode memnode;
 
-u16 x86_cpu_to_node_map_init[NR_CPUS] = {
+int x86_cpu_to_node_map_init[NR_CPUS] = {
 	[0 ... NR_CPUS-1] = NUMA_NO_NODE
 };
 void *x86_cpu_to_node_map_early_ptr;
 EXPORT_SYMBOL(x86_cpu_to_node_map_init);
 EXPORT_SYMBOL(x86_cpu_to_node_map_early_ptr);
-DEFINE_PER_CPU(u16, x86_cpu_to_node_map) = NUMA_NO_NODE;
+DEFINE_PER_CPU(int, x86_cpu_to_node_map) = NUMA_NO_NODE;
 EXPORT_PER_CPU_SYMBOL(x86_cpu_to_node_map);
 
-u16 apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
+s16 apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
 	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
 };
 

@@ -64,7 +64,7 @@ static int __init populate_memnodemap(co
 	unsigned long addr, end;
 	int i, res = -1;
 
-	memset(memnodemap, 0xff, memnodemapsize);
+	memset(memnodemap, 0xff, sizeof(s16)*memnodemapsize);
 	for (i = 0; i < numnodes; i++) {
 		addr = nodes[i].start;
 		end = nodes[i].end;

@@ -73,7 +73,7 @@ static int __init populate_memnodemap(co
 		if ((end >> shift) >= memnodemapsize)
 			return 0;
 		do {
-			if (memnodemap[addr >> shift] != 0xff)
+			if (memnodemap[addr >> shift] != NUMA_NO_NODE)
 				return -1;
 			memnodemap[addr >> shift] = i;
 			addr += (1UL << shift);

@@ -88,7 +88,7 @@ static int __init allocate_cachealigned_
 	unsigned long pad, pad_addr;
 
 	memnodemap = memnode.embedded_map;
-	if (memnodemapsize <= 48)
+	if (memnodemapsize <= ARRAY_SIZE(memnode.embedded_map))
 		return 0;
 
 	pad = L1_CACHE_BYTES - 1;

@@ -566,7 +566,7 @@ __cpuinit void numa_add_cpu(int cpu)
 
 void __cpuinit numa_set_node(int cpu, int node)
 {
-	u16 *cpu_to_node_map = x86_cpu_to_node_map_early_ptr;
+	int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr;
 
 	cpu_pda(cpu)->nodenumber = node;
include/asm-x86/mmzone_64.h +3 -3
@@ -15,9 +15,9 @@
 struct memnode {
 	int shift;
 	unsigned int mapsize;
-	u8 *map;
-	u8 embedded_map[64-16];
-} ____cacheline_aligned; /* total size = 64 bytes */
+	s16 *map;
+	s16 embedded_map[64-8];
+} ____cacheline_aligned; /* total size = 128 bytes */
 extern struct memnode memnode;
 #define memnode_shift memnode.shift
 #define memnodemap memnode.map
include/asm-x86/numa_64.h +1 -1
@@ -20,7 +20,7 @@ extern void numa_set_node(int cpu, int n
 extern void srat_reserve_add_area(int nodeid);
 extern int hotadd_percent;
 
-extern u16 apicid_to_node[MAX_LOCAL_APIC];
+extern s16 apicid_to_node[MAX_LOCAL_APIC];
 
 extern void numa_initmem_init(unsigned long start_pfn, unsigned long end_pfn);
 extern unsigned long numa_free_all_bootmem(void);
include/asm-x86/topology.h +4 -4
@@ -30,17 +30,17 @@
 #include <asm/mpspec.h>
 
 /* Mappings between logical cpu number and node number */
-DECLARE_PER_CPU(u16, x86_cpu_to_node_map);
-extern u16 x86_cpu_to_node_map_init[];
+DECLARE_PER_CPU(int, x86_cpu_to_node_map);
+extern int x86_cpu_to_node_map_init[];
 extern void *x86_cpu_to_node_map_early_ptr;
 extern cpumask_t node_to_cpumask_map[];
 
-#define NUMA_NO_NODE	((u16)(~0))
+#define NUMA_NO_NODE	(-1)
 
 /* Returns the number of the node containing CPU 'cpu' */
 static inline int cpu_to_node(int cpu)
 {
-	u16 *cpu_to_node_map = x86_cpu_to_node_map_early_ptr;
+	int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr;
 
 	if (cpu_to_node_map)
 		return cpu_to_node_map[cpu];
Signature · 5 lines
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
author
#2 lore raw

PATCH 2/4 x86: Change NR_CPUS arrays in numa_64 fixup V2

In reply to [email protected]

Change the following static arrays sized by NR_CPUS to per_cpu data variables:

	char cpu_to_node_map[NR_CPUS];

Based on 2.6.24-rc8-mm1

Signed-off-by Mike Travis <[email protected]>
Reviewed-by Christoph Lameter <[email protected]>

---
fixup:

- Split cpu_to_node function into "early" and "late" versions so that x86_cpu_to_node_map_early_ptr is not EXPORT'ed and the cpu_to_node inline function is more streamlined.

- This also involves setting up the percpu maps as early as possible.

- Fix X86_32 NUMA build errors that previous version of this patch caused.

V2->V3:
- add early_cpu_to_node function to keep cpu_to_node efficient
- move and rename smp_set_apicids() to setup_percpu_maps()
- call setup_percpu_maps() as early as possible

V1->V2:
- Removed extraneous casts
- Fix !NUMA builds with '#ifdef CONFIG_NUMA"

---
 arch/x86/kernel/setup64.c    |   41 +++++++++++++++++++++++++++++++++++++++--
 arch/x86/kernel/smpboot_64.c |   34 ----------------------------------
 arch/x86/mm/numa_64.c        |    2 --
 arch/x86/mm/srat_64.c        |    7 ++++---
 include/asm-x86/topology.h   |   23 +++++++++++++++++++++++
 5 files changed, 66 insertions(+), 41 deletions(-)
arch/x86/kernel/setup64.c +39 -2
@@ -84,6 +84,40 @@ static int __init nonx32_setup(char *str
 __setup("noexec32=", nonx32_setup);
 
 /*
+ * Copy data used in early init routines from the initial arrays to the
+ * per cpu data areas.  These arrays then become expendable and the
+ * *_early_ptr's are zeroed indicating that the static arrays are gone.
+ */
+static void __init setup_per_cpu_maps(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+#ifdef CONFIG_SMP
+		if (per_cpu_offset(cpu)) {
+#endif
+			per_cpu(x86_cpu_to_apicid, cpu) =
+						x86_cpu_to_apicid_init[cpu];
+#ifdef CONFIG_NUMA
+			per_cpu(x86_cpu_to_node_map, cpu) =
+						x86_cpu_to_node_map_init[cpu];
+#endif
+#ifdef CONFIG_SMP
+		}
+		else
+			printk(KERN_NOTICE "per_cpu_offset zero for cpu %d\n",
+									cpu);
+#endif
+	}
+
+	/* indicate the early static arrays will soon be gone */
+	x86_cpu_to_apicid_early_ptr = NULL;
+#ifdef CONFIG_NUMA
+	x86_cpu_to_node_map_early_ptr = NULL;
+#endif
+}
+
+/*
  * Great future plan:
  * Declare PDA itself and support (irqstack,tss,pgd) as per cpu data.
  * Always point %gs to its beginning

@@ -104,18 +138,21 @@ void __init setup_per_cpu_areas(void)
 	for_each_cpu_mask (i, cpu_possible_map) {
 		char *ptr;
 
-		if (!NODE_DATA(cpu_to_node(i))) {
+		if (!NODE_DATA(early_cpu_to_node(i))) {
 			printk("cpu with no node %d, num_online_nodes %d\n",
 			       i, num_online_nodes());
 			ptr = alloc_bootmem_pages(size);
 		} else { 
-			ptr = alloc_bootmem_pages_node(NODE_DATA(cpu_to_node(i)), size);
+			ptr = alloc_bootmem_pages_node(NODE_DATA(early_cpu_to_node(i)), size);
 		}
 		if (!ptr)
 			panic("Cannot allocate cpu data for CPU %d\n", i);
 		cpu_pda(i)->data_offset = ptr - __per_cpu_start;
 		memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
 	}
+
+	/* setup percpu data maps early */
+	setup_per_cpu_maps();
 } 
 
 void pda_init(int cpu)
arch/x86/kernel/smpboot_64.c +0 -34
@@ -851,39 +851,6 @@ static int __init smp_sanity_check(unsig
 	return 0;
 }
 
-/*
- * Copy data used in early init routines from the initial arrays to the
- * per cpu data areas.  These arrays then become expendable and the
- * *_ptrs are zeroed indicating that the static arrays are gone.
- */
-void __init smp_set_apicids(void)
-{
-	int cpu;
-
-	for_each_possible_cpu(cpu) {
-		if (per_cpu_offset(cpu)) {
-			per_cpu(x86_cpu_to_apicid, cpu) =
-						x86_cpu_to_apicid_init[cpu];
-#ifdef CONFIG_NUMA
-			per_cpu(x86_cpu_to_node_map, cpu) =
-						x86_cpu_to_node_map_init[cpu];
-#endif
-			per_cpu(x86_bios_cpu_apicid, cpu) =
-						x86_bios_cpu_apicid_init[cpu];
-		}
-		else
-			printk(KERN_NOTICE "per_cpu_offset zero for cpu %d\n",
-									cpu);
-	}
-
-	/* indicate the early static arrays are gone */
-	x86_cpu_to_apicid_early_ptr = NULL;
-#ifdef CONFIG_NUMA
-	x86_cpu_to_node_map_early_ptr = NULL;
-#endif
-	x86_bios_cpu_apicid_early_ptr = NULL;
-}
-
 static void __init smp_cpu_index_default(void)
 {
 	int i;

@@ -906,7 +873,6 @@ void __init smp_prepare_cpus(unsigned in
 	smp_cpu_index_default();
 	current_cpu_data = boot_cpu_data;
 	current_thread_info()->cpu = 0;  /* needed? */
-	smp_set_apicids();
 	set_cpu_sibling_map(0);
 
 	if (smp_sanity_check(max_cpus) < 0) {
arch/x86/mm/numa_64.c +0 -2
@@ -35,8 +35,6 @@ int x86_cpu_to_node_map_init[NR_CPUS] = 
 	[0 ... NR_CPUS-1] = NUMA_NO_NODE
 };
 void *x86_cpu_to_node_map_early_ptr;
-EXPORT_SYMBOL(x86_cpu_to_node_map_init);
-EXPORT_SYMBOL(x86_cpu_to_node_map_early_ptr);
 DEFINE_PER_CPU(int, x86_cpu_to_node_map) = NUMA_NO_NODE;
 EXPORT_PER_CPU_SYMBOL(x86_cpu_to_node_map);
arch/x86/mm/srat_64.c +4 -3
@@ -382,9 +382,10 @@ int __init acpi_scan_nodes(unsigned long
 			setup_node_bootmem(i, nodes[i].start, nodes[i].end);
 
 	for (i = 0; i < NR_CPUS; i++) {
-		if (cpu_to_node(i) == NUMA_NO_NODE)
+		int node = early_cpu_to_node(i);
+		if (node == NUMA_NO_NODE)
 			continue;
-		if (!node_isset(cpu_to_node(i), node_possible_map))
+		if (!node_isset(node, node_possible_map))
 			numa_set_node(i, NUMA_NO_NODE);
 	}
 	numa_init_array();

@@ -395,7 +396,7 @@ int __init acpi_scan_nodes(unsigned long
 static int fake_node_to_pxm_map[MAX_NUMNODES] __initdata = {
 	[0 ... MAX_NUMNODES-1] = PXM_INVAL
 };
-static u16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
+static s16 fake_apicid_to_node[MAX_LOCAL_APIC] __initdata = {
 	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
 };
 static int __init find_node_by_addr(unsigned long addr)
include/asm-x86/topology.h +23 -0
@@ -30,16 +30,30 @@
 #include <asm/mpspec.h>
 
 /* Mappings between logical cpu number and node number */
+#ifdef CONFIG_X86_32
+extern int cpu_to_node_map[];
+
+#else
 DECLARE_PER_CPU(int, x86_cpu_to_node_map);
 extern int x86_cpu_to_node_map_init[];
 extern void *x86_cpu_to_node_map_early_ptr;
+#endif
+
 extern cpumask_t node_to_cpumask_map[];
 
 #define NUMA_NO_NODE	(-1)
 
 /* Returns the number of the node containing CPU 'cpu' */
+#ifdef CONFIG_X86_32
+#define early_cpu_to_node(cpu)	cpu_to_node(cpu)
 static inline int cpu_to_node(int cpu)
 {
+	return cpu_to_node_map[cpu];
+}
+
+#else /* CONFIG_X86_64 */
+static inline int early_cpu_to_node(int cpu)
+{
 	int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr;
 
 	if (cpu_to_node_map)

@@ -50,6 +64,15 @@ static inline int cpu_to_node(int cpu)
 		return NUMA_NO_NODE;
 }
 
+static inline int cpu_to_node(int cpu)
+{
+	if(per_cpu_offset(cpu))
+		return per_cpu(x86_cpu_to_node_map, cpu);
+	else
+		return NUMA_NO_NODE;
+}
+#endif /* CONFIG_X86_64 */
+
 /*
  * Returns the number of the node containing Node 'node'. This
  * architecture is flat, so it is a pretty simple function!
Signature · 5 lines
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
author
#3 lore raw

PATCH 3/4 x86: Change bios_cpu_apicid to percpu data variable fixup V2

In reply to [email protected]

Change static bios_cpu_apicid array to a per_cpu data variable. This includes using a static array used during initialization similar to the way x86_cpu_to_apicid[] is handled.

There is one early use of bios_cpu_apicid in apic_is_clustered_box(). The other reference in cpu_present_to_apicid() is called after setup_per_cpu_maps() has setup the percpu version of bios_cpu_apicid.

Based on 2.6.24-rc8-mm1

Signed-off-by Mike Travis <[email protected]>
Reviewed-by Christoph Lameter <[email protected]>

---
V1->V2:
- Removed extraneous casts
- Add slight optimization to apic_is_clustered_box() [don't reference x86_bios_cpu_apicid_early_ptr each pass.]

---
 arch/x86/kernel/apic_64.c  |    6 +++---
 arch/x86/kernel/setup64.c  |    3 +++
 arch/x86/kernel/setup_64.c |    1 +
 3 files changed, 7 insertions(+), 3 deletions(-)
arch/x86/kernel/apic_64.c +3 -3
@@ -1191,9 +1191,9 @@ __cpuinit int apic_is_clustered_box(void
 
 	/* Problem:  Partially populated chassis may not have CPUs in some of
 	 * the APIC clusters they have been allocated.  Only present CPUs have
-	 * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap.  Since
-	 * clusters are allocated sequentially, count zeros only if they are
-	 * bounded by ones.
+	 * x86_bios_cpu_apicid entries, thus causing zeroes in the bitmap.
+	 * Since clusters are allocated sequentially, count zeros only if
+	 * they are bounded by ones.
 	 */
 	clusters = 0;
 	zeros = 0;
arch/x86/kernel/setup64.c +3 -0
@@ -98,6 +98,8 @@ static void __init setup_per_cpu_maps(vo
 #endif
 			per_cpu(x86_cpu_to_apicid, cpu) =
 						x86_cpu_to_apicid_init[cpu];
+			per_cpu(x86_bios_cpu_apicid, cpu) =
+						x86_bios_cpu_apicid_init[cpu];
 #ifdef CONFIG_NUMA
 			per_cpu(x86_cpu_to_node_map, cpu) =
 						x86_cpu_to_node_map_init[cpu];

@@ -112,6 +114,7 @@ static void __init setup_per_cpu_maps(vo
 
 	/* indicate the early static arrays will soon be gone */
 	x86_cpu_to_apicid_early_ptr = NULL;
+	x86_bios_cpu_apicid_early_ptr = NULL;
 #ifdef CONFIG_NUMA
 	x86_cpu_to_node_map_early_ptr = NULL;
 #endif
arch/x86/kernel/setup_64.c +1 -0
@@ -390,6 +390,7 @@ void __init setup_arch(char **cmdline_p)
 #ifdef CONFIG_SMP
 	/* setup to use the early static init tables during kernel startup */
 	x86_cpu_to_apicid_early_ptr = (void *)&x86_cpu_to_apicid_init;
+	x86_bios_cpu_apicid_early_ptr = (void *)&x86_bios_cpu_apicid_init;
 #ifdef CONFIG_NUMA
 	x86_cpu_to_node_map_early_ptr = (void *)&x86_cpu_to_node_map_init;
 #endif
Signature · 5 lines
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
author
#4 lore raw

PATCH 4/4 x86: Add debug of invalid per_cpu map accesses fixup V2

In reply to [email protected]

Provide a means to discover usages of per_cpu map variables before they are setup. Define CONFIG_DEBUG_PER_CPU_MAPS to activate.

Based on 2.6.24-rc8-mm1

Signed-off-by Mike Travis <[email protected]>

---
Fixup:
- for cpu_to_node() instead of panic'ing with BUG() use dump_stack and return valid node id.

---
 arch/x86/Kconfig.debug     |   12 ++++++++++++
 arch/x86/mm/numa_64.c      |    3 +++
 include/asm-x86/topology.h |   12 ++++++++++--
 3 files changed, 25 insertions(+), 2 deletions(-)
arch/x86/Kconfig.debug +12 -0
@@ -47,6 +47,18 @@ config DEBUG_PAGEALLOC
 	  This results in a large slowdown, but helps to find certain types
 	  of memory corruptions.
 
+config DEBUG_PER_CPU_MAPS
+	bool "Debug access to per_cpu maps"
+	depends on DEBUG_KERNEL
+	depends on X86_64_SMP
+	default n
+	help
+	  Say Y to verify that the per_cpu map being accessed has
+	  been setup.  Adds a fair amount of code to kernel memory
+	  and decreases performance.
+
+	  Say N if unsure.
+
 config DEBUG_RODATA
 	bool "Write protect kernel read-only data structures"
 	depends on DEBUG_KERNEL
arch/x86/mm/numa_64.c +3 -0
@@ -37,6 +37,9 @@ int x86_cpu_to_node_map_init[NR_CPUS] = 
 void *x86_cpu_to_node_map_early_ptr;
 DEFINE_PER_CPU(int, x86_cpu_to_node_map) = NUMA_NO_NODE;
 EXPORT_PER_CPU_SYMBOL(x86_cpu_to_node_map);
+#ifdef	CONFIG_DEBUG_PER_CPU_MAPS
+EXPORT_SYMBOL(x86_cpu_to_node_map_early_ptr);
+#endif
 
 s16 apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = {
 	[0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE
include/asm-x86/topology.h +10 -2
@@ -58,7 +58,7 @@ static inline int early_cpu_to_node(int 
 
 	if (cpu_to_node_map)
 		return cpu_to_node_map[cpu];
-	else if(per_cpu_offset(cpu))
+	else if (per_cpu_offset(cpu))
 		return per_cpu(x86_cpu_to_node_map, cpu);
 	else
 		return NUMA_NO_NODE;

@@ -66,7 +66,15 @@ static inline int early_cpu_to_node(int 
 
 static inline int cpu_to_node(int cpu)
 {
-	if(per_cpu_offset(cpu))
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+	if (x86_cpu_to_node_map_early_ptr) {
+		printk("KERN_NOTICE cpu_to_node(%d): usage too early!\n",
+			(int)cpu);
+		dump_stack();
+		return ((int *)x86_cpu_to_node_map_early_ptr)[cpu];
+	}
+#endif
+	if (per_cpu_offset(cpu))
 		return per_cpu(x86_cpu_to_node_map, cpu);
 	else
 		return NUMA_NO_NODE;
Signature · 5 lines
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>