patch to /proc/meminfo to display NUMA stats

RFC In review Martin J. Bligh 4 participants 6 messages Apr 22, 2002 – Apr 24, 2002

Subsystems (6)

Memory management MM core Page allocator Proc filesystem +2 more subsystems

lore

Share & tools

lore

Martin J. Bligh

author

2002-04-22 16:07 UTC #0 lore raw

RFC 1/1 patch to /proc/meminfo to display NUMA stats

Below is a patch to /proc/meminfo to display free, used and total memory per node on a NUMA machine. It works fine on an ia32 machine, but is not yet ready for submission until I make it generic. Before I go to the effort of doing that, I thought I'd seek some feedback.

Comments?

Thanks,

Martin.

arch/i386/mm/init.c +12 -0

diff -urN virgin-2.4.18/arch/i386/mm/init.c linux-2.4.18-meminfo/arch/i386/mm/init.c
@@ -598,6 +598,18 @@
 	return;
 }
 
+#ifdef CONFIG_NUMA
+void si_meminfo_node(struct sysinfo *val, int nid)
+{
+	val->totalram = NODE_DATA(nid)->node_size;
+	val->freeram = nr_free_pages_node(nid);
+	val->totalhigh = NODE_DATA(nid)->node_zones[ZONE_HIGHMEM].size;
+	val->freehigh = nr_free_highpages_node(nid);
+	val->mem_unit = PAGE_SIZE;
+	return;
+}
+#endif
+
 #if defined(CONFIG_X86_PAE)
 struct kmem_cache_s *pae_pgd_cachep;
 void __init pgtable_cache_init(void)

fs/proc/proc_misc.c +22 -1

diff -urN virgin-2.4.18/fs/proc/proc_misc.c linux-2.4.18-meminfo/fs/proc/proc_misc.c
@@ -132,7 +132,7 @@
 				 int count, int *eof, void *data)
 {
 	struct sysinfo i;
-	int len;
+	int len, nid;
 	int pg_size ;
 
 /*

@@ -185,6 +185,27 @@
 		K(i.freeram-i.freehigh),
 		K(i.totalswap),
 		K(i.freeswap));
+
+#ifdef CONFIG_NUMA
+	for (nid = 0; nid < numnodes; ++nid) {
+		si_meminfo_node(&i, nid);
+		len += sprintf(page+len, "\n"
+			"Node %d MemTotal:     %8lu kB\n"
+			"Node %d MemFree:     %8lu kB\n"
+			"Node %d MemUsed:     %8lu kB\n"
+			"Node %d HighTotal:    %8lu kB\n"
+			"Node %d HighFree:     %8lu kB\n"
+			"Node %d LowTotal:     %8lu kB\n"
+			"Node %d LowFree:      %8lu kB\n",
+			nid, K(i.totalram),
+			nid, K(i.freeram),
+			nid, K(i.totalram-i.freeram),
+			nid, K(i.totalhigh),
+			nid, K(i.freehigh),
+			nid, K(i.totalram-i.totalhigh),
+			nid, K(i.freeram-i.freehigh));
+	}
+#endif
 
 	return proc_calc_metrics(page, start, off, count, eof, len);
 #undef B

include/linux/highmem.h +3 -0

diff -urN virgin-2.4.18/include/linux/highmem.h linux-2.4.18-meminfo/include/linux/highmem.h
@@ -12,6 +12,9 @@
 
 /* declarations for linux/mm/highmem.c */
 unsigned int nr_free_highpages(void);
+#ifdef CONFIG_NUMA
+unsigned int nr_free_highpages_node (int);
+#endif
 
 extern struct buffer_head * create_bounce(int rw, struct buffer_head * bh_orig);

include/linux/mm.h +3 -0

diff -urN virgin-2.4.18/include/linux/mm.h linux-2.4.18-meminfo/include/linux/mm.h
@@ -447,6 +447,9 @@
 extern void mem_init(void);
 extern void show_mem(void);
 extern void si_meminfo(struct sysinfo * val);
+#ifdef CONFIG_NUMA
+extern void si_meminfo_node(struct sysinfo *val, int nid);
+#endif
 extern void swapin_readahead(swp_entry_t);
 
 extern struct address_space swapper_space;

include/linux/swap.h +3 -0

diff -urN virgin-2.4.18/include/linux/swap.h linux-2.4.18-meminfo/include/linux/swap.h
@@ -84,6 +84,9 @@
 #define vm_swap_full() (nr_swap_pages*2 < total_swap_pages)
 
 extern unsigned int nr_free_pages(void);
+#ifdef CONFIG_NUMA
+extern unsigned int nr_free_pages_node (int);
+#endif
 extern unsigned int nr_free_buffer_pages(void);
 extern int nr_active_pages;
 extern int nr_inactive_pages;

mm/page_alloc.c +25 -1

diff -urN virgin-2.4.18/mm/page_alloc.c linux-2.4.18-meminfo/mm/page_alloc.c
@@ -462,6 +462,23 @@
 	return sum;
 }
 
+#ifdef CONFIG_NUMA
+/*
+ * Total amount of free (allocatable) RAM for a given node:
+ */
+unsigned int nr_free_pages_node (int nid)
+{
+	unsigned int sum;
+	zone_t *zone;
+	pg_data_t *pgdat = NODE_DATA(nid);
+
+	sum = 0;
+	for (zone = pgdat->node_zones; zone < pgdat->node_zones + MAX_NR_ZONES; zone++)
+		sum += zone->free_pages;
+	return sum;
+}
+#endif
+
 /*
  * Amount of free RAM allocatable as buffer memory:
  */

@@ -500,7 +517,14 @@
 	}
 	return pages;
 }
-#endif
+
+#ifdef CONFIG_NUMA
+unsigned int nr_free_highpages_node (int nid)
+{
+	return NODE_DATA(nid)->node_zones[ZONE_HIGHMEM].free_pages;
+}
+#endif /* CONFIG_NUMA */
+#endif /* CONFIG_HIGHMEM */
 
 #define K(x) ((x) << (PAGE_SHIFT-10))

Christoph Hellwig

2002-04-22 16:40 UTC #1 lore raw

Quoted 6 lines · Martin J. Bligh

Below is a patch to /proc/meminfo to display free, used and total memory per node on a NUMA machine. It works fine on an ia32 machine, but is not yet ready for submission until I make it generic. Before I go to the effort of doing that, I thought I'd seek some feedback.
Comments?

Sure. First I wonder whether it would be possible to kill nr_free_pages() and nr_free_highpages() (or infact all of the nr_free*page() functions) in favour of the per-node ones, externalizing the for loop. Or if there are to many users at least make them use the per-node version. Second I wonder whether nr_free_pages_node() and nr_free_highpages_node() should take a pgdat instead of a node id as the caller has it around anyway.

Also I think the per-node info should be in /proc/meminfo even for UMA machines, it doesn't hurt but makes it more consistant.

Martin J. Bligh

author

2002-04-24 18:42 UTC #2 lore raw

OK, here version 2

Quoted 6 lines

Sure. First I wonder whether it would be possible to kill nr_free_pages() and nr_free_highpages() (or infact all of the nr_free*page() functions) in favour of the per-node ones, externalizing the for loop. Or if there are to many users at least make them use the per-node version. Second I wonder whether nr_free_pages_node() and nr_free_highpages_node() should take a pgdat instead of a node id as the caller has it around anyway.

OK, I've taken most of this - I made the nr_free_pages extensions into _pgdat calls that operate on a pgdat, as you suggested. I still think we need a nr_free_pages and friends to give global statistics, but I made those call the pgdat functions (which I made inline to keep the speed up - they're small with not many callers, so hopefully this isn't controversial).

Quoted 2 lines

Also I think the per-node info should be in /proc/meminfo even for UMA machines, it doesn't hurt but makes it more consistant.

I'm slightly wary of doing this, as it'll break tools that parse /proc/meminfo for everybody ... what I'm doing is fairly conservative, and having the global section still there seems useful to me for the NUMA machines too.

I also shifted the si_meminfo_node into mm/numa.c to make it generic for all machines, and made it loop through the node_next loop of pgdats to make it work on machines that have discontig within a node.

What do people think of the attatched version?

fs/proc/proc_misc.c +22 -1

diff -urN virgin-2.4.18/fs/proc/proc_misc.c linux-2.4.18-meminfo/fs/proc/proc_misc.c
@@ -132,7 +132,7 @@
 				 int count, int *eof, void *data)
 {
 	struct sysinfo i;
-	int len;
+	int len, nid;
 	int pg_size ;
 
 /*

@@ -185,6 +185,27 @@
 		K(i.freeram-i.freehigh),
 		K(i.totalswap),
 		K(i.freeswap));
+
+#ifdef CONFIG_NUMA
+	for (nid = 0; nid < numnodes; ++nid) {
+		si_meminfo_node(&i, nid);
+		len += sprintf(page+len, "\n"
+			"Node %d MemTotal:     %8lu kB\n"
+			"Node %d MemFree:     %8lu kB\n"
+			"Node %d MemUsed:     %8lu kB\n"
+			"Node %d HighTotal:    %8lu kB\n"
+			"Node %d HighFree:     %8lu kB\n"
+			"Node %d LowTotal:     %8lu kB\n"
+			"Node %d LowFree:      %8lu kB\n",
+			nid, K(i.totalram),
+			nid, K(i.freeram),
+			nid, K(i.totalram-i.freeram),
+			nid, K(i.totalhigh),
+			nid, K(i.freehigh),
+			nid, K(i.totalram-i.totalhigh),
+			nid, K(i.freeram-i.freehigh));
+	}
+#endif
 
 	return proc_calc_metrics(page, start, off, count, eof, len);
 #undef B

include/linux/highmem.h +1 -0

diff -urN virgin-2.4.18/include/linux/highmem.h linux-2.4.18-meminfo/include/linux/highmem.h
@@ -12,6 +12,7 @@
 
 /* declarations for linux/mm/highmem.c */
 unsigned int nr_free_highpages(void);
+inline unsigned int nr_free_highpages_pgdat(pg_data_t *pgdat);
 
 extern struct buffer_head * create_bounce(int rw, struct buffer_head * bh_orig);

include/linux/mm.h +3 -0

diff -urN virgin-2.4.18/include/linux/mm.h linux-2.4.18-meminfo/include/linux/mm.h
@@ -447,6 +447,9 @@
 extern void mem_init(void);
 extern void show_mem(void);
 extern void si_meminfo(struct sysinfo * val);
+#ifdef CONFIG_NUMA
+extern void si_meminfo_node(struct sysinfo *val, int nid);
+#endif
 extern void swapin_readahead(swp_entry_t);
 
 extern struct address_space swapper_space;

include/linux/swap.h +1 -0

diff -urN virgin-2.4.18/include/linux/swap.h linux-2.4.18-meminfo/include/linux/swap.h
@@ -84,6 +84,7 @@
 #define vm_swap_full() (nr_swap_pages*2 < total_swap_pages)
 
 extern unsigned int nr_free_pages(void);
+extern inline unsigned int nr_free_pages_pgdat (pg_data_t *pgdat);
 extern unsigned int nr_free_buffer_pages(void);
 extern int nr_active_pages;
 extern int nr_inactive_pages;

mm/numa.c +23 -0

diff -urN virgin-2.4.18/mm/numa.c linux-2.4.18-meminfo/mm/numa.c
@@ -128,3 +128,26 @@
 }
 
 #endif /* CONFIG_DISCONTIGMEM */
+
+void si_meminfo_node(struct sysinfo *val, int nid)
+{
+	pg_data_t *pgdat;
+
+	val->totalram = 0;
+	val->freeram = 0;
+	val->totalhigh = 0;
+	val->freehigh = 0;
+
+	pgdat = NODE_DATA(nid);
+	while (pgdat) {
+		val->totalram = pgdat->node_size;
+		val->freeram = nr_free_pages_pgdat(pgdat);
+		#ifdef CONFIG_HIGHMEM
+			val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].size;
+			val->freehigh = nr_free_highpages_pgdat(pgdat);
+		#endif
+		pgdat = pgdat->node_next;
+	}
+	val->mem_unit = PAGE_SIZE;
+	return;
+}

mm/page_alloc.c +22 -5

diff -urN virgin-2.4.18/mm/page_alloc.c linux-2.4.18-meminfo/mm/page_alloc.c
@@ -450,19 +450,31 @@
 unsigned int nr_free_pages (void)
 {
 	unsigned int sum;
-	zone_t *zone;
 	pg_data_t *pgdat = pgdat_list;
 
 	sum = 0;
 	while (pgdat) {
-		for (zone = pgdat->node_zones; zone < pgdat->node_zones + MAX_NR_ZONES; zone++)
-			sum += zone->free_pages;
+		sum += nr_free_pages_pgdat(pgdat);
 		pgdat = pgdat->node_next;
 	}
 	return sum;
 }
 
 /*
+ * Total amount of free (allocatable) RAM for a given pg_data_t:
+ */
+inline unsigned int nr_free_pages_pgdat (pg_data_t *pgdat)
+{
+	unsigned int sum; 
+	zone_t *zone;
+
+	sum = 0;
+	for (zone = pgdat->node_zones; zone < pgdat->node_zones + MAX_NR_ZONES; zone++)
+		sum += zone->free_pages;
+	return sum;
+}
+
+/*
  * Amount of free RAM allocatable as buffer memory:
  */
 unsigned int nr_free_buffer_pages (void)

@@ -495,12 +507,17 @@
 	unsigned int pages = 0;
 
 	while (pgdat) {
-		pages += pgdat->node_zones[ZONE_HIGHMEM].free_pages;
+		pages += nr_free_highpages_pgdat(pgdat);
 		pgdat = pgdat->node_next;
 	}
 	return pages;
 }
-#endif
+
+inline unsigned int nr_free_highpages_pgdat (pg_data_t *pgdat)
+{
+	return pgdat->node_zones[ZONE_HIGHMEM].free_pages;
+}
+#endif /* CONFIG_HIGHMEM */
 
 #define K(x) ((x) << (PAGE_SHIFT-10))

Martin J. Bligh

author

2002-04-24 18:49 UTC #3 lore raw

Quoted 3 lines

I also shifted the si_meminfo_node into mm/numa.c to make it generic for all machines, and made it loop through the node_next loop of pgdats to make it work on machines that have discontig within a node.

Sigh ... of course si_meminfo_node should have done += to the elements within the loop, not =. Sorry,

Keith Owens

2002-04-22 23:39 UTC #4 lore raw

In reply to Martin J. Bligh

Quoted 18 lines · Martin J. Bligh

Below is a patch to /proc/meminfo to display free, used and total memory per node on a NUMA machine. It works fine on an ia32 machine, but is not yet ready for submission until I make it generic. Before I go to the effort of doing that, I thought I'd seek some feedback.
diff -urN virgin-2.4.18/fs/proc/proc_misc.c linux-2.4.18-meminfo/fs/proc/proc_misc.c
fs/proc/proc_misc.c +10 -0
+#ifdef CONFIG_NUMA
+	for (nid = 0; nid < numnodes; ++nid) {
+		si_meminfo_node(&i, nid);
+		len += sprintf(page+len, "\n"
+			"Node %d MemTotal:     %8lu kB\n"
+			"Node %d MemFree:     %8lu kB\n"
+			"Node %d MemUsed:     %8lu kB\n"
+			"Node %d HighTotal:    %8lu kB\n"
+			"Node %d HighFree:     %8lu kB\n"
+			"Node %d LowTotal:     %8lu kB\n"

fs/proc/proc_misc.c line 208

Quoted 10 lines

fs/proc/proc_misc.c +10 -0

+			"Node %d LowFree:      %8lu kB\n",
+			nid, K(i.totalram),
+			nid, K(i.freeram),
+			nid, K(i.totalram-i.freeram),
+			nid, K(i.totalhigh),
+			nid, K(i.freehigh),
+			nid, K(i.totalram-i.totalhigh),
+			nid, K(i.freeram-i.freehigh));
+	}
+#endif

meminfo_read_proc() assumes that all its output will fit in one page. Currently it does, ~520 bytes. You are adding ~120 bytes per node so it will break at ~29 nodes using 4K pages. To make that scalable, meminfo_read_proc() must be converted to seq_file format.

Jesse Barnes

2002-04-23 17:29 UTC #5 lore raw

In reply to Martin J. Bligh

I implemented something kind of like this for the discontig project, but I called it /proc/discontig. The patch just walked all the pgdata_t structures and did its best to find out where the memory was being used. See http://sourceforge.net/projects/discontig for the patch. The first column of the output should probably be called 'region' or something, since it's not NUMA specific at all...

Jesse

Quoted 30 lines · Keith Owens

Quoted 28 lines · Martin J. Bligh
Below is a patch to /proc/meminfo to display free, used and total memory per node on a NUMA machine. It works fine on an ia32 machine, but is not yet ready for submission until I make it generic. Before I go to the effort of doing that, I thought I'd seek some feedback.
diff -urN virgin-2.4.18/fs/proc/proc_misc.c linux-2.4.18-meminfo/fs/proc/proc_misc.c
fs/proc/proc_misc.c +20 -0
+#ifdef CONFIG_NUMA
+	for (nid = 0; nid < numnodes; ++nid) {
+		si_meminfo_node(&i, nid);
+		len += sprintf(page+len, "\n"
+			"Node %d MemTotal:     %8lu kB\n"
+			"Node %d MemFree:     %8lu kB\n"
+			"Node %d MemUsed:     %8lu kB\n"
+			"Node %d HighTotal:    %8lu kB\n"
+			"Node %d HighFree:     %8lu kB\n"
+			"Node %d LowTotal:     %8lu kB\n"
+			"Node %d LowFree:      %8lu kB\n",
+			nid, K(i.totalram),
+			nid, K(i.freeram),
+			nid, K(i.totalram-i.freeram),
+			nid, K(i.totalhigh),
+			nid, K(i.freehigh),
+			nid, K(i.totalram-i.totalhigh),
+			nid, K(i.freeram-i.freehigh));
+	}
+#endif