Skip to contentSkip to conversation

: fake numa for x86_64 patches

Patch3/4 Rohit Seth 2 participants 4 messages Nov 23, 2006 – Nov 27, 2006

lore
Share & tools
lore

Rohit Seth

author
#0 lore raw

Fix the existing numa=fake so that ioholes are appropriately configured. Currently machines that have sizeable IO holes don't work with numa=fake>4. This patch tries to equally partition the total available memory in equal size chunk. The minimum size of the fake node is set to 32MB.

include/asm-x86_64/mmzone.h +15 -14
@@ -11,25 +11,26 @@
 
 #include <asm/smp.h>
 
-/* Should really switch to dynamic allocation at some point */
-#define NODEMAPSIZE 0x4fff
-
-/* Simple perfect hash to map physical addresses to node numbers */
-struct memnode {
-	int shift;
-	u8 map[NODEMAPSIZE];
-} ____cacheline_aligned;
-extern struct memnode memnode;
-#define memnode_shift memnode.shift
-#define memnodemap memnode.map
+/*
+ * Generic node memory support with the following assumptions:
+ * 1) Memory is allocated in NODE_MIN_SIZE contiguous chunks
+ * 2) There is not more than 64GB of system memory total
+ *
+ * 64GB / (4KB / page) == 16777216 pages
+ */
+#define MAX_NR_PAGES 16777216
+#define NODE_MIN_SIZE (32<<20)
+#define PAGES_PER_ELEMENT (NODE_MIN_SIZE / PAGE_SIZE)
+#define MAX_ELEMENTS (MAX_NR_PAGES / PAGES_PER_ELEMENT)
+#define NODE_HASH_MASK (~(NODE_MIN_SIZE -1ul))
 
+extern unsigned char physnode_map[];
 extern struct pglist_data *node_data[];
 
 static inline __attribute__((pure)) int phys_to_nid(unsigned long addr) 
 { 
-	unsigned nid; 
-	VIRTUAL_BUG_ON((addr >> memnode_shift) >= NODEMAPSIZE);
-	nid = memnodemap[addr >> memnode_shift]; 
+	int nid; 
+	nid = ((int)physnode_map[(addr >> PAGE_SHIFT) / PAGES_PER_ELEMENT]);
 	VIRTUAL_BUG_ON(nid >= MAX_NUMNODES || !node_data[nid]); 
 	return nid; 
 } 
include/asm-x86_64/numa.h +1 -1
@@ -7,7 +7,7 @@ struct bootnode {
 	u64 start,end; 
 };
 
-extern int compute_hash_shift(struct bootnode *nodes, int numnodes);
+extern void populate_physnode_map(struct bootnode *nodes, int numnodes);
 
 #define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT))
 
arch/x86_64/mm/k8topology.c +5 -7
@@ -141,7 +141,10 @@ int __init k8_scan_nodes(unsigned long s
 			       prevbase,base);
 			return -1;
 		}
-			
+		
+		/* Round down to NODE_HASH_MASK */
+		limit = ((limit - base) & NODE_HASH_MASK) + base;
+
 		printk(KERN_INFO "Node %d MemBase %016lx Limit %016lx\n", 
 		       nodeid, base, limit); 
 		
@@ -161,12 +164,7 @@ int __init k8_scan_nodes(unsigned long s
 	if (!found)
 		return -1; 
 
-	memnode_shift = compute_hash_shift(nodes, 8);
-	if (memnode_shift < 0) { 
-		printk(KERN_ERR "No NUMA node hash function found. Contact maintainer\n"); 
-		return -1; 
-	} 
-	printk(KERN_INFO "Using node hash shift of %d\n", memnode_shift); 
+	populate_physnode_map(nodes, 8);
 
 	for (i = 0; i < 8; i++) {
 		if (nodes[i].start != nodes[i].end) { 
arch/x86_64/mm/srat.c +5 -12
@@ -31,10 +31,6 @@ static struct bootnode nodes_add[MAX_NUM
 static int found_add_area __initdata;
 int hotadd_percent __initdata = 0;
 
-/* Too small nodes confuse the VM badly. Usually they result
-   from BIOS bugs. */
-#define NODE_MIN_SIZE (4*1024*1024)
-
 static __init int setup_node(int pxm)
 {
 	return acpi_map_pxm_to_node(pxm);
@@ -394,11 +390,14 @@ int __init acpi_scan_nodes(unsigned long
 
 	/* First clean up the node list */
 	for (i = 0; i < MAX_NUMNODES; i++) {
+		signed long sz = nodes[i].end - nodes[i].start;
  		cutoff_node(i, start, end);
-		if ((nodes[i].end - nodes[i].start) < NODE_MIN_SIZE) {
+		if (sz < NODE_MIN_SIZE) {
 			unparse_node(i);
 			node_set_offline(i);
 		}
+		/* Round down. */
+		nodes[i].end = nodes[i].start + (sz & NODE_HASH_MASK);
 	}
 
 	if (acpi_numa <= 0)
@@ -409,13 +408,7 @@ int __init acpi_scan_nodes(unsigned long
 		return -1;
 	}
 
-	memnode_shift = compute_hash_shift(nodes, MAX_NUMNODES);
-	if (memnode_shift < 0) {
-		printk(KERN_ERR
-		     "SRAT: No NUMA node hash function found. Contact maintainer\n");
-		bad_srat();
-		return -1;
-	}
+	populate_physnode_map(nodes, MAX_NUMNODES);
 
 	/* Finally register nodes */
 	for_each_node_mask(i, nodes_parsed)
arch/x86_64/mm/numa.c +143 -88

293 lines not on this page · raw message

#1 lore raw
Quoted 5 lines · Rohit Seth

Fix the existing numa=fake so that ioholes are appropriately configured. Currently machines that have sizeable IO holes don't work with numa=fake>4. This patch tries to equally partition the total available memory in equal size chunk. The minimum size of the fake node is set to 32MB.

This patch seems to do far more than advertised in the change log?

You're conflicting badly with Amul's numa hash function rewrite for example.

I don't see why you need to change that much code for this relatively simple change anyways.

-Andi

author
#2 lore raw
Quoted 11 lines · Andi Kleen
Quoted 5 lines · Rohit Seth

Fix the existing numa=fake so that ioholes are appropriately configured. Currently machines that have sizeable IO holes don't work with numa=fake>4. This patch tries to equally partition the total available memory in equal size chunk. The minimum size of the fake node is set to 32MB.

This patch seems to do far more than advertised in the change log?

You're conflicting badly with Amul's numa hash function rewrite for example.

Both of these patches are mucking with hash function and populate_memnodemap. I like Amul's approach of doing dynamic allocation of numa hash map so that it can support >64GB of memory space. I will resend the patches on top of his patch (incorporating your other feedback).

-rohit

#3 lore raw
Quoted 10 lines · Rohit Seth
Quoted 9 lines · Andi Kleen
Quoted 5 lines · Rohit Seth

Fix the existing numa=fake so that ioholes are appropriately configured. Currently machines that have sizeable IO holes don't work with numa=fake>4. This patch tries to equally partition the total available memory in equal size chunk. The minimum size of the fake node is set to 32MB.

This patch seems to do far more than advertised in the change log?

Quoted 8 lines
Quoted 2 lines

You're conflicting badly with Amul's numa hash function rewrite for example.

Both of these patches are mucking with hash function and populate_memnodemap. I like Amul's approach of doing dynamic allocation of numa hash map so that it can support >64GB of memory space. I will resend the patches on top of his patch (incorporating your other feedback).

FYI I dropped Amul's patch temporarily because it causes boot failures on some systems. But it will be likely readded once that problem is fixed.

-Andi