Skip to contentSkip to conversation

NUMA memory configuration issue

PATCH In review Zoltan Menyhart 2 participants 5 messages Apr 10, 2008 – Apr 16, 2008

lore
Share & tools
lore

Zoltan Menyhart

author
#0 lore raw

PATCH 1/1 NUMA memory configuration issue

There is a NUMA memory configuration issue in 2.6.24:

A 2-node machine of ours has got the following memory layout:

Node 0: 0 - 2 Gbytes
Node 0: 4 - 8 Gbytes
Node 1: 8 - 16 Gbytes
Node 0: 16 - 18 Gbytes

"efi_memmap_init()" merges the three last ranges into one.

"register_active_ranges()" is called as follows:

	efi_memmap_walk(register_active_ranges, NULL);

i.e. once for the 4 - 18 Gbytes range. It picks up the node number from the start address, and registers all the memory for the node #0.

"register_active_ranges()" should be called as follows to make sure there is no merged address range at its entry:

	efi_memmap_walk(filter__memory, register_active_ranges);

"filter__memory()" is similar to "filter_rsvd_memory()", but the reserved memory ranges are not filtered out.

Thanks,

Zoltan Menyhart

Signed-off-by Zoltan Menyhart, <[email protected]>
arch/ia64/kernel/setup.c +21 -0
diff -Nru linux-2.6.24.4/arch/ia64/kernel/setup.c linux-2.6.24.4-test/arch/ia64/kernel/setup.c
@@ -178,6 +178,27 @@
 	return 0;
 }
 
+/*
+ * Similar to "filter_rsvd_memory()", but the reserved memory ranges are not filtered out.
+ */
+int __init
+filter__memory (unsigned long start, unsigned long end, void *arg)
+{
+	void (*func)(unsigned long, unsigned long, int);
+
+#if IGNORE_PFN0
+	if (start == PAGE_OFFSET) {
+		printk(KERN_WARNING "warning: skipping physical page 0\n");
+		start += PAGE_SIZE;
+		if (start >= end) return 0;
+	}
+#endif
+	func = arg;
+	if (start < end)
+		call_pernode_memory(__pa(start), end - start, func);
+	return 0;
+}
+
 static void __init
 sort_regions (struct rsvd_region *rsvd_region, int max)
 {
arch/ia64/mm/discontig.c +1 -1
diff -Nru linux-2.6.24.4/arch/ia64/mm/discontig.c linux-2.6.24.4-test/arch/ia64/mm/discontig.c
@@ -444,7 +444,7 @@
 			mem_data[node].min_pfn = ~0UL;
 		}
 
-	efi_memmap_walk(register_active_ranges, NULL);
+	efi_memmap_walk(filter__memory, register_active_ranges);
 
 	/*
 	 * Initialize the boot memory maps in reverse order since that's
arch/ia64/mm/init.c +2 -4
diff -Nru linux-2.6.24.4/arch/ia64/mm/init.c linux-2.6.24.4-test/arch/ia64/mm/init.c
@@ -553,12 +553,10 @@
 #endif /* CONFIG_VIRTUAL_MEM_MAP */
 
 int __init
-register_active_ranges(u64 start, u64 end, void *arg)
+register_active_ranges(u64 start, u64 len, int nid)
 {
-	int nid = paddr_to_nid(__pa(start));
+	u64 end = start + len;
 
-	if (nid < 0)
-		nid = 0;
 #ifdef CONFIG_KEXEC
 	if (start > crashk_res.start && start < crashk_res.end)
 		start = crashk_res.end;
include/asm-ia64/meminit.h +2 -1
diff -Nru linux-2.6.24.4/include/asm-ia64/meminit.h linux-2.6.24.4-test/include/asm-ia64/meminit.h
@@ -35,6 +35,7 @@
 extern void reserve_memory (void);
 extern void find_initrd (void);
 extern int filter_rsvd_memory (unsigned long start, unsigned long end, void *arg);
+extern int filter__memory (unsigned long start, unsigned long end, void *arg);
 extern unsigned long efi_memmap_init(unsigned long *s, unsigned long *e);
 extern int find_max_min_low_pfn (unsigned long , unsigned long, void *);
 

@@ -56,7 +57,7 @@
 
 #define IGNORE_PFN0	1	/* XXX fix me: ignore pfn 0 until TLB miss handler is updated... */
 
-extern int register_active_ranges(u64 start, u64 end, void *arg);
+extern int register_active_ranges(u64 start, u64 len, int nid);
 
 #ifdef CONFIG_VIRTUAL_MEM_MAP
 # define LARGE_GAP	0x40000000 /* Use virtual mem map if hole is > than this */
#1 lore raw

You missed a spot:

arch/ia64/mm/contig.c +1 -1
index 0479661..798bf98 100644
@@ -253,7 +253,7 @@ paging_init (void)
 	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
 
 #ifdef CONFIG_VIRTUAL_MEM_MAP
-	efi_memmap_walk(register_active_ranges, NULL);
+	efi_memmap_walk(filter_memory, register_active_ranges);
 	efi_memmap_walk(find_largest_hole, (u64 *)&max_gap);
 	if (max_gap < LARGE_GAP) {
 		vmem_map = (struct page *) 0;

But otherwise this looks plausible. My non-numa system still boots.

-Tony

author
#2 lore raw
In reply to Zoltan Menyhart
Quoted 17 lines · Tony

You missed a spot:

arch/ia64/mm/contig.c +1 -1
index 0479661..798bf98 100644
@@ -253,7 +253,7 @@ paging_init (void)
 	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
 
 #ifdef CONFIG_VIRTUAL_MEM_MAP
-	efi_memmap_walk(register_active_ranges, NULL);
+	efi_memmap_walk(filter_memory, register_active_ranges);
 	efi_memmap_walk(find_largest_hole, (u64 *)&max_gap);
 	if (max_gap < LARGE_GAP) {
 		vmem_map = (struct page *) 0;
Quoted 2 lines

But otherwise this looks plausible. My non-numa system still boots.

Thank your for your response.

My patch aims to avoid the confusion due to the EFI memory ranges which belong to different numa nodes and are merged in certain conditions.

Do you think contig.c is also effected?
Can you please explain why?

Thanks,

Zoltan

#3 lore raw
In reply to Zoltan Menyhart
Quoted 2 lines

Do you think contig.c is also effected?
Can you please explain why?

No action is required in the contig.c case, but the compiler grumbles because you changed the type (of the arguments) for register_active_ranges(). So for a .config with CONFIG_VIRTUAL_MEM_MAP=y the following warning is produced:

CC arch/ia64/mm/contig.o
arch/ia64/mm/contig.c: In function `paging_init':
arch/ia64/mm/contig.c:258: warning: passing arg 1 of `efi_memmap_walk' from incompatible pointer type

-Tony

author
#4 lore raw
In reply to Zoltan Menyhart
Quoted 10 lines · Tony

No action is required in the contig.c case, but the compiler grumbles because you changed the type (of the arguments) for register_active_ranges(). So for a .config with CONFIG_VIRTUAL_MEM_MAP=y the following warning is produced:

CC arch/ia64/mm/contig.o
arch/ia64/mm/contig.c: In function `paging_init':
arch/ia64/mm/contig.c:258: warning: passing arg 1 of `efi_memmap_walk' from incompatible pointer type

-Tony

Thank you.

There is a NUMA memory configuration issue in 2.6.24:

A 2-node machine of ours has got the following memory layout:

Node 0:    0 - 2 Gbytes
Node 0:    4 - 8 Gbytes
Node 1:    8 - 16 Gbytes
Node 0:    16 - 18 Gbytes

"efi_memmap_init()" merges the three last ranges into one.

"register_active_ranges()" is called as follows:

   efi_memmap_walk(register_active_ranges, NULL);

i.e. once for the 4 - 18 Gbytes range. It picks up the node number from the start address, and registers all the memory for the node #0.

"register_active_ranges()" should be called as follows to make sure there is no merged address range at its entry:

   efi_memmap_walk(filter__memory, register_active_ranges);

"filter__memory()" is similar to "filter_rsvd_memory()", but the reserved memory ranges are not filtered out.

Thanks,

Zoltan Menyhart

Signed-off-by Zoltan Menyhart, <[email protected]>
arch/ia64/kernel/setup.c +21 -0
diff -Nru linux-2.6.24.4/arch/ia64/kernel/setup.c linux-2.6.24.4-test/arch/ia64/kernel/setup.c
@@ -178,6 +178,27 @@
 	return 0;
 }
 
+/*
+ * Similar to "filter_rsvd_memory()", but the reserved memory ranges are not filtered out.
+ */
+int __init
+filter__memory (unsigned long start, unsigned long end, void *arg)
+{
+	void (*func)(unsigned long, unsigned long, int);
+
+#if IGNORE_PFN0
+	if (start == PAGE_OFFSET) {
+		printk(KERN_WARNING "warning: skipping physical page 0\n");
+		start += PAGE_SIZE;
+		if (start >= end) return 0;
+	}
+#endif
+	func = arg;
+	if (start < end)
+		call_pernode_memory(__pa(start), end - start, func);
+	return 0;
+}
+
 static void __init
 sort_regions (struct rsvd_region *rsvd_region, int max)
 {
arch/ia64/mm/contig.c +1 -1
diff -Nru linux-2.6.24.4/arch/ia64/mm/contig.c linux-2.6.24.4-test/arch/ia64/mm/contig.c
@@ -255,7 +255,7 @@
 	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
 
 #ifdef CONFIG_VIRTUAL_MEM_MAP
-	efi_memmap_walk(register_active_ranges, NULL);
+	efi_memmap_walk(filter__memory, register_active_ranges);
 	efi_memmap_walk(find_largest_hole, (u64 *)&max_gap);
 	if (max_gap < LARGE_GAP) {
 		vmem_map = (struct page *) 0;
arch/ia64/mm/discontig.c +1 -1
diff -Nru linux-2.6.24.4/arch/ia64/mm/discontig.c linux-2.6.24.4-test/arch/ia64/mm/discontig.c
@@ -444,7 +444,7 @@
 			mem_data[node].min_pfn = ~0UL;
 		}
 
-	efi_memmap_walk(register_active_ranges, NULL);
+	efi_memmap_walk(filter__memory, register_active_ranges);
 
 	/*
 	 * Initialize the boot memory maps in reverse order since that's
arch/ia64/mm/init.c +2 -4
diff -Nru linux-2.6.24.4/arch/ia64/mm/init.c linux-2.6.24.4-test/arch/ia64/mm/init.c
@@ -553,12 +553,10 @@
 #endif /* CONFIG_VIRTUAL_MEM_MAP */
 
 int __init
-register_active_ranges(u64 start, u64 end, void *arg)
+register_active_ranges(u64 start, u64 len, int nid)
 {
-	int nid = paddr_to_nid(__pa(start));
+	u64 end = start + len;
 
-	if (nid < 0)
-		nid = 0;
 #ifdef CONFIG_KEXEC
 	if (start > crashk_res.start && start < crashk_res.end)
 		start = crashk_res.end;
include/asm-ia64/meminit.h +2 -1
diff -Nru linux-2.6.24.4/include/asm-ia64/meminit.h linux-2.6.24.4-test/include/asm-ia64/meminit.h
@@ -35,6 +35,7 @@
 extern void reserve_memory (void);
 extern void find_initrd (void);
 extern int filter_rsvd_memory (unsigned long start, unsigned long end, void *arg);
+extern int filter__memory (unsigned long start, unsigned long end, void *arg);
 extern unsigned long efi_memmap_init(unsigned long *s, unsigned long *e);
 extern int find_max_min_low_pfn (unsigned long , unsigned long, void *);
 

@@ -56,7 +57,7 @@
 
 #define IGNORE_PFN0	1	/* XXX fix me: ignore pfn 0 until TLB miss handler is updated... */
 
-extern int register_active_ranges(u64 start, u64 end, void *arg);
+extern int register_active_ranges(u64 start, u64 len, int nid);
 
 #ifdef CONFIG_VIRTUAL_MEM_MAP
 # define LARGE_GAP	0x40000000 /* Use virtual mem map if hole is > than this */