sparsemem: chop up the global mem_map[]

RFC PATCH In review Dave Hansen 1 participant 1 message Mar 17, 2005

Subsystems (5)

Memory management MM core Generic include/asm header files Page allocator +1 more subsystems

lore

Share & tools

lore

Dave Hansen

author

2005-03-17 00:28 UTC #0 lore raw

RFC PATCH 1/6 sparsemem: chop up the global mem_map[]

Sparsemem abstracts the use of discontiguous mem_maps[]. This kind of mem_map[] is needed by discontiguous memory machines (like in the old CONFIG_DISCONTIGMEM case) as well as memory hotplug systems. Sparsemem replaces DISCONTIGMEM when enabled, and it is hoped that it can eventually become a complete replacement.

A significant advantage over DISCONTIGMEM is that it's completely separated from CONFIG_NUMA. When producing this patch, it became apparent in that NUMA and DISCONTIG are often confused.

Another advantage is that sparse doesn't require each NUMA node's ranges to be contiguous. It can handle overlapping ranges between nodes with no problems, where DISCONTIGMEM currently throws away that memory.

Sparsemem uses an array to provide different pfn_to_page() translations for each SECTION_SIZE area of physical memory. This is what allows the mem_map[] to be chopped up.

In order to do quick pfn_to_page() operations, the section number of the page is encoded in page->flags. Part of the sparsemem infrastructure enables sharing of these bits more dynamically (at compile-time) between the page_zone() and sparsemem operations. However, on 32-bit architectures, the number of bits is quite limited, and may require growing the size of the page->flags type in certain conditions. Several things might force this to
occur: a decrease in the SECTION_SIZE (if you want to hotplug smaller areas of memory), an increase in the physical address space, or an increase in the number of used page->flags.

One thing to note is that, once sparsemem is present, the NUMA node information no longer needs to be stored in the page->flags. It might provide speed increases on certain platforms and will be stored there if there is room. But, if out of room, an alternate (theoretically slower) mechanism is used.

This patch introduces CONFIG_FLATMEM. It is used in almost all cases where there used to be an #ifndef DISCONTIG, because SPARSEMEM and DISCONTIGMEM often have to compile out the same areas of code.

Another addition is mm/Kconfig. With sparsemem and memory hotplug there are quite a few options that we kept adding identically in several different architectures. The new file allows some of these to be consolidated.

In this series is a i386. However, there also exist ppc64, ia64, and x86_64 patches. Expect these to come via the architecture maintainers very soon.

Signed-off-by Andy Whitcroft <[email protected]>

Signed-off-by Dave Hansen <[email protected]>

---

 arch/i386/Kconfig                        |    0 
 memhotplug-dave/arch/i386/mm/discontig.c |    2 
 memhotplug-dave/include/linux/mm.h       |   86 +++++++++++++++++++++++-----
 memhotplug-dave/include/linux/mmzone.h   |   94 ++++++++++++++++++++++++++++++-
 memhotplug-dave/include/linux/numa.h     |    2 
 memhotplug-dave/mm/Kconfig               |   53 +++++++++++++++++
 memhotplug-dave/mm/Makefile              |    1 
 memhotplug-dave/mm/bootmem.c             |    8 +-
 memhotplug-dave/mm/memory.c              |    2 
 memhotplug-dave/mm/page_alloc.c          |   33 ++++++++--
 memhotplug-dave/mm/sparse.c              |   87 ++++++++++++++++++++++++++++
 11 files changed, 340 insertions(+), 28 deletions(-)

arch/i386/mm/discontig.c renamed from arch/i386/mm/discontig.c~B-sparse-150-sparsemem +2 -0

diff -puN arch/i386/mm/discontig.c~B-sparse-150-sparsemem arch/i386/mm/discontig.c
@@ -46,6 +46,8 @@ bootmem_data_t node0_bdata;
  * 3) node_start_pfn   - the starting page frame number for a node
  * 3) node_end_pfn     - the ending page fram number for a node
  */
+unsigned long node_start_pfn[MAX_NUMNODES];
+unsigned long node_end_pfn[MAX_NUMNODES];
 
 /*
  * physnode_map keeps track of the physical memory layout of a generic

include/linux/mm.h renamed from include/linux/mm.h~B-sparse-150-sparsemem +72 ~~-14~~

123 lines not on this page · raw message

include/linux/mmzone.h renamed from include/linux/mmzone.h~B-sparse-150-sparsemem +91 -3

115 lines not on this page · raw message

include/linux/numa.h renamed from include/linux/numa.h~B-sparse-150-sparsemem +1 -1

diff -puN include/linux/numa.h~B-sparse-150-sparsemem include/linux/numa.h
@@ -3,7 +3,7 @@
 
 #include <linux/config.h>
 
-#ifdef CONFIG_DISCONTIGMEM
+#ifndef CONFIG_FLATMEM
 #include <asm/numnodes.h>
 #endif

mm/Makefile renamed from mm/Makefile~B-sparse-150-sparsemem +1 -0

diff -puN mm/Makefile~B-sparse-150-sparsemem mm/Makefile
@@ -15,6 +15,7 @@ obj-y			:= bootmem.o filemap.o mempool.o
 obj-$(CONFIG_SWAP)	+= page_io.o swap_state.o swapfile.o thrash.o
 obj-$(CONFIG_HUGETLBFS)	+= hugetlb.o
 obj-$(CONFIG_NUMA) 	+= mempolicy.o
+obj-$(CONFIG_SPARSEMEM)	+= sparse.o
 obj-$(CONFIG_SHMEM) += shmem.o
 obj-$(CONFIG_TINY_SHMEM) += tiny-shmem.o

mm/bootmem.c renamed from mm/bootmem.c~B-sparse-150-sparsemem +6 -2

diff -puN mm/bootmem.c~B-sparse-150-sparsemem mm/bootmem.c
@@ -256,6 +256,7 @@ found:
 static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
 {
 	struct page *page;
+	unsigned long pfn;
 	bootmem_data_t *bdata = pgdat->bdata;
 	unsigned long i, count, total = 0;
 	unsigned long idx;

@@ -266,7 +267,7 @@ static unsigned long __init free_all_boo
 
 	count = 0;
 	/* first extant page of the node */
-	page = virt_to_page(phys_to_virt(bdata->node_boot_start));
+	pfn = bdata->node_boot_start >> PAGE_SHIFT;
 	idx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT);
 	map = bdata->node_bootmem_map;
 	/* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */

@@ -275,6 +276,9 @@ static unsigned long __init free_all_boo
 		gofast = 1;
 	for (i = 0; i < idx; ) {
 		unsigned long v = ~map[i / BITS_PER_LONG];
+
+		page = pfn_to_page(pfn);
+
 		if (gofast && v == ~0UL) {
 			int j, order;

@@ -302,8 +306,8 @@ static unsigned long __init free_all_boo
 			}
 		} else {
 			i+=BITS_PER_LONG;
-			page += BITS_PER_LONG;
 		}
+		pfn += BITS_PER_LONG;
 	}
 	total += count;

mm/memory.c renamed from mm/memory.c~B-sparse-150-sparsemem +1 -1

diff -puN mm/memory.c~B-sparse-150-sparsemem mm/memory.c
@@ -58,7 +58,7 @@
 #include <linux/swapops.h>
 #include <linux/elf.h>
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 /* use the per-pgdat data instead for discontigmem - mbligh */
 unsigned long max_mapnr;
 struct page *mem_map;

mm/page_alloc.c renamed from mm/page_alloc.c~B-sparse-150-sparsemem +26 -7

diff -puN mm/page_alloc.c~B-sparse-150-sparsemem mm/page_alloc.c
@@ -66,7 +66,7 @@ EXPORT_SYMBOL(nr_swap_pages);
  * Used by page_zone() to look up the address of the struct zone whose
  * id is encoded in the upper bits of page->flags
  */
-struct zone *zone_table[1 << (ZONES_SHIFT + NODES_SHIFT)];
+struct zone *zone_table[1 << ZONETABLE_SHIFT];
 EXPORT_SYMBOL(zone_table);
 
 static char *zone_names[MAX_NR_ZONES] = { "DMA", "Normal", "HighMem" };

@@ -1579,11 +1579,15 @@ static void __init calculate_zone_totalp
 void __init memmap_init_zone(unsigned long size, int nid, unsigned long zone,
 		unsigned long start_pfn)
 {
-	struct page *start = pfn_to_page(start_pfn);
 	struct page *page;
+	int end_pfn = start_pfn + size;
+	int pfn;
 
-	for (page = start; page < (start + size); page++) {
-		set_page_links(page, zone, nid);
+	for (pfn = start_pfn; pfn < end_pfn; pfn++, page++) {
+		if (!early_pfn_valid(pfn))
+			continue;
+		page = pfn_to_page(pfn);
+		set_page_links(page, zone, nid, pfn);
 		set_page_count(page, 0);
 		reset_page_mapcount(page);
 		SetPageReserved(page);

@@ -1607,6 +1611,20 @@ void zone_init_free_lists(struct pglist_
 	}
 }
 
+#define ZONETABLE_INDEX(x, zone_nr)	((x << ZONES_SHIFT) | zone_nr)
+void zonetable_add(struct zone *zone, int nid, int zid, unsigned long pfn,
+		unsigned long size)
+{
+	unsigned long snum = pfn >> PFN_SECTION_SHIFT;
+	unsigned long end = (pfn + size) >> PFN_SECTION_SHIFT;
+
+	if (FLAGS_HAS_NODE)
+		zone_table[ZONETABLE_INDEX(nid, zid)] = zone;
+	else
+		for (; snum <= end; snum++)
+			zone_table[ZONETABLE_INDEX(snum, zid)] = zone;
+}
+
 #ifndef __HAVE_ARCH_MEMMAP_INIT
 #define memmap_init(size, nid, zone, start_pfn) \
 	memmap_init_zone((size), (nid), (zone), (start_pfn))

@@ -1635,7 +1653,6 @@ static void __init free_area_init_core(s
 		unsigned long size, realsize;
 		unsigned long batch;
 
-		zone_table[NODEZONE(nid, j)] = zone;
 		realsize = size = zones_size[j];
 		if (zholes_size)
 			realsize -= zholes_size[j];

@@ -1720,6 +1737,8 @@ static void __init free_area_init_core(s
 
 		memmap_init(size, nid, j, zone_start_pfn);
 
+		zonetable_add(zone, nid, j, zone_start_pfn, size);
+
 		zone_start_pfn += size;
 
 		zone_init_free_lists(pgdat, zone, zone->spanned_pages);

@@ -1743,7 +1762,7 @@ static void __init alloc_node_mem_map(st
 			map = alloc_bootmem_node(pgdat, size);
 		pgdat->node_mem_map = map;
 	}
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 	/*
 	 * With no DISCONTIG, the global mem_map is just set as node 0's
 	 */

@@ -1765,7 +1784,7 @@ void __init free_area_init_node(int nid,
 	free_area_init_core(pgdat, zones_size, zholes_size);
 }
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 static bootmem_data_t contig_bootmem_data;
 struct pglist_data contig_page_data = { .bdata = &contig_bootmem_data };

mm/sparse.c new file +87 -0

87 lines not on this page · raw message

mm/Kconfig new file +53 -0

53 lines not on this page · raw message

diff -puN arch/i386/Kconfig~B-sparse-150-sparsemem arch/i386/Kconfig _