Skip to contentSkip to conversation

quieten NUMA boot messages

PATCH New Anton Blanchard 1 participant 2 messages Sep 2, 2004

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 quieten NUMA boot messages

On some machines we would print hundreds of lines of NUMA debug output. The following patch cleans it up so we only print a summary of cpus and memory vs nodes:

Node 0 CPUs: 0-1
Node 1 CPUs: 16-17
Node 2 CPUs: 32-33
Node 3 CPUs: 48-49
Node 0 Memory: 0x0-0x400000000
Node 1 Memory: 0x400000000-0x800000000

I lifted the code to do this out of xmon.

arch/ppc64/mm/numa.c renamed from arch/ppc64/mm/numa.c~quieten_numa +78 -23
diff -puN arch/ppc64/mm/numa.c~quieten_numa arch/ppc64/mm/numa.c
@@ -18,11 +18,8 @@
 #include <asm/machdep.h>
 #include <asm/abs_addr.h>
 
-#if 1
-#define dbg(args...) printk(KERN_INFO args)
-#else
-#define dbg(args...)
-#endif
+static int numa_debug;
+#define dbg(args...) if (numa_debug) { printk(KERN_INFO args); }
 
 #ifdef DEBUG_NUMA
 #define ARRAY_INITIALISER -1
@@ -48,7 +45,6 @@ EXPORT_SYMBOL(nr_cpus_in_node);
 
 static inline void map_cpu_to_node(int cpu, int node)
 {
-	dbg("cpu %d maps to domain %d\n", cpu, node);
 	numa_cpu_lookup_table[cpu] = node;
 	if (!(cpu_isset(cpu, numa_cpumask_lookup_table[node]))) {
 		cpu_set(cpu, numa_cpumask_lookup_table[node]);
@@ -107,8 +103,8 @@ static int of_node_numa_domain(struct de
 	if (tmp && (tmp[0] >= depth)) {
 		numa_domain = tmp[depth];
 	} else {
-		printk(KERN_ERR "WARNING: no NUMA information for "
-		       "%s\n", device->full_name);
+		dbg("WARNING: no NUMA information for %s\n",
+		    device->full_name);
 		numa_domain = 0;
 	}
 	return numa_domain;
@@ -137,11 +133,8 @@ static int find_min_common_depth(void)
 
 	rtas_root = of_find_node_by_path("/rtas");
 
-	if (!rtas_root) {
-		printk(KERN_ERR "WARNING: %s() could not find rtas root\n",
-				__FUNCTION__);
+	if (!rtas_root)
 		return -1;
-	}
 
 	/*
 	 * this property is 2 32-bit integers, each representing a level of
@@ -155,8 +148,8 @@ static int find_min_common_depth(void)
 	if ((len >= 1) && ref_points) {
 		depth = ref_points[1];
 	} else {
-		printk(KERN_ERR "WARNING: could not find NUMA "
-				"associativity reference point\n");
+		dbg("WARNING: could not find NUMA "
+		    "associativity reference point\n");
 		depth = -1;
 	}
 	of_node_put(rtas_root);
@@ -187,6 +180,9 @@ static int __init parse_numa_properties(
 	long entries = lmb_end_of_DRAM() >> MEMORY_INCREMENT_SHIFT;
 	unsigned long i;
 
+	if (strstr(saved_command_line, "numa=debug"))
+		numa_debug = 1;
+
 	if (strstr(saved_command_line, "numa=off")) {
 		printk(KERN_WARNING "NUMA disabled by user\n");
 		return -1;
@@ -200,7 +196,7 @@ static int __init parse_numa_properties(
 
 	depth = find_min_common_depth();
 
-	printk(KERN_INFO "NUMA associativity depth for CPU/Memory: %d\n", depth);
+	dbg("NUMA associativity depth for CPU/Memory: %d\n", depth);
 	if (depth < 0)
 		return depth;
 
@@ -225,8 +221,7 @@ static int __init parse_numa_properties(
 				numa_domain = 0;
 			}
 		} else {
-			printk(KERN_ERR "WARNING: no NUMA information for "
-			       "cpu %ld\n", i);
+			dbg("WARNING: no NUMA information for cpu %ld\n", i);
 			numa_domain = 0;
 		}
 
@@ -286,9 +281,9 @@ new_range:
 				node_data[numa_domain].node_start_pfn + 
 				node_data[numa_domain].node_spanned_pages;
 			if (shouldstart != (start / PAGE_SIZE)) {
-				printk(KERN_ERR "Hole in node, disabling "
-						"region start %lx length %lx\n",
-						start, size);
+				printk(KERN_ERR "WARNING: Hole in node, "
+						"disabling region start %lx "
+						"length %lx\n", start, size);
 				continue;
 			}
 			node_data[numa_domain].node_spanned_pages +=
@@ -304,9 +299,6 @@ new_range:
 			numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] =
 				numa_domain;
 
-		dbg("memory region %lx to %lx maps to domain %d\n",
-		    start, start+size, numa_domain);
-
 		ranges--;
 		if (ranges)
 			goto new_range;
@@ -350,6 +342,67 @@ static void __init setup_nonnuma(void)
 	node0_io_hole_size = top_of_ram - total_ram;
 }
 
+static void __init dump_numa_topology(void)
+{
+	unsigned int node;
+	unsigned int cpu, count;
+
+	for (node = 0; node < MAX_NUMNODES; node++) {
+		if (!node_online(node))
+			continue;
+
+		printk(KERN_INFO "Node %d CPUs:", node);
+
+		count = 0;
+		/*
+		 * If we used a CPU iterator here we would miss printing
+		 * the holes in the cpumap.
+		 */
+		for (cpu = 0; cpu < NR_CPUS; cpu++) {
+			if (cpu_isset(cpu, numa_cpumask_lookup_table[node])) {
+				if (count == 0)
+					printk(" %u", cpu);
+				++count;
+			} else {
+				if (count > 1)
+					printk("-%u", cpu - 1);
+				count = 0;
+			}
+		}
+
+		if (count > 1)
+			printk("-%u", NR_CPUS - 1);
+		printk("\n");
+	}
+
+	for (node = 0; node < MAX_NUMNODES; node++) {
+		unsigned long i;
+
+		if (!node_online(node))
+			continue;
+
+		printk(KERN_INFO "Node %d Memory:", node);
+
+		count = 0;
+
+		for (i = 0; i < lmb_end_of_DRAM(); i += MEMORY_INCREMENT) {
+			if (numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] == node) {
+				if (count == 0)
+					printk(" 0x%lx", i);
+				++count;
+			} else {
+				if (count > 0)
+					printk("-0x%lx", i);
+				count = 0;
+			}
+		}
+
+		if (count > 0)
+			printk("-0x%lx", i);
+		printk("\n");
+	}
+}
+
 void __init do_init_bootmem(void)
 {
 	int nid;
@@ -360,6 +413,8 @@ void __init do_init_bootmem(void)
 
 	if (parse_numa_properties())
 		setup_nonnuma();
+	else
+		dump_numa_topology();
 
 	for (nid = 0; nid < numnodes; nid++) {
 		unsigned long start_paddr, end_paddr;

_

#1 lore raw

PATCH 1/1 Allocate NUMA node data node locally

Instead of statically allocating the NUMA node structures, do it at runtime with node local memory where possible. With NR_CPUS=128 we had over 800kB of memory allocated before this patch, now we allocate only as many nodes as the machine requires.

We have to do some fancy footwork in do_init_bootmem since we use both the LMB allocator and the bootmem allocator at the same time. The problem has always been there although I only recently found a test case that hit it. Wrap that logic in careful_allocation which uses the LMB allocator then falls back to the bootmem allocator if required.

Now alloc_bootmem on a node with no memory doesnt panic, we dont need the hack in paging_init so remove it.

arch/ppc64/mm/init.c renamed from arch/ppc64/mm/init.c~bootmem_fixes +1 -1
diff -puN arch/ppc64/mm/init.c~bootmem_fixes arch/ppc64/mm/init.c
@@ -663,7 +663,7 @@ void __init mem_init(void)
 	int nid;
 
         for (nid = 0; nid < numnodes; nid++) {
-		if (node_data[nid].node_spanned_pages != 0) {
+		if (NODE_DATA(nid)->node_spanned_pages != 0) {
 			printk("freeing bootmem node %x\n", nid);
 			totalram_pages +=
 				free_all_bootmem_node(NODE_DATA(nid));
arch/ppc64/mm/numa.c renamed from arch/ppc64/mm/numa.c~bootmem_fixes +88 -32

186 lines not on this page · raw message

include/asm-ppc64/mmzone.h renamed from include/asm-ppc64/mmzone.h~bootmem_fixes +2 -2
diff -puN include/asm-ppc64/mmzone.h~bootmem_fixes include/asm-ppc64/mmzone.h
@@ -12,7 +12,7 @@
 
 #ifdef CONFIG_DISCONTIGMEM
 
-extern struct pglist_data node_data[];
+extern struct pglist_data *node_data[];
 
 /*
  * Following are specific to this numa platform.
@@ -52,7 +52,7 @@ static inline int pa_to_nid(unsigned lon
 /*
  * Return a pointer to the node data for node n.
  */
-#define NODE_DATA(nid)		(&node_data[nid])
+#define NODE_DATA(nid)		(node_data[nid])
 
 #define node_localnr(pfn, nid)	((pfn) - NODE_DATA(nid)->node_start_pfn)
 

_