050 bootmem use NODE_DATA

diff -upN reference/mm/bootmem.c current/mm/bootmem.c
@@ -343,31 +343,29 @@ unsigned long __init free_all_bootmem_no
 	return(free_all_bootmem_core(pgdat));
 }
 
-#ifndef CONFIG_DISCONTIGMEM
 unsigned long __init init_bootmem (unsigned long start, unsigned long pages)
 {
 	max_low_pfn = pages;
 	min_low_pfn = start;
-	return(init_bootmem_core(&contig_page_data, start, 0, pages));
+	return(init_bootmem_core(NODE_DATA(0), start, 0, pages));
 }
 
 #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
 void __init reserve_bootmem (unsigned long addr, unsigned long size)
 {
-	reserve_bootmem_core(contig_page_data.bdata, addr, size);
+	reserve_bootmem_core(NODE_DATA(0)->bdata, addr, size);
 }
 #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
 
 void __init free_bootmem (unsigned long addr, unsigned long size)
 {
-	free_bootmem_core(contig_page_data.bdata, addr, size);
+	free_bootmem_core(NODE_DATA(0)->bdata, addr, size);
 }
 
 unsigned long __init free_all_bootmem (void)
 {
-	return(free_all_bootmem_core(&contig_page_data));
+	return(free_all_bootmem_core(NODE_DATA(0)));
 }
-#endif /* !CONFIG_DISCONTIGMEM */
 
 void * __init __alloc_bootmem (unsigned long size, unsigned long align, unsigned long goal)
 {

diff -upN reference/arch/i386/kernel/setup.c current/arch/i386/kernel/setup.c
@@ -941,8 +941,6 @@ unsigned long __init find_max_low_pfn(vo
 	return max_low_pfn;
 }
 
-#ifndef CONFIG_DISCONTIGMEM
-
 /*
  * Free all available memory for boot time allocation.  Used
  * as a callback function by efi_memory_walk()

@@ -1016,15 +1014,15 @@ static void __init reserve_ebda_region(v
 		reserve_bootmem(addr, PAGE_SIZE);	
 }
 
+#ifndef CONFIG_DISCONTIGMEM
+void __init setup_bootmem_allocator(void);
 static unsigned long __init setup_memory(void)
 {
-	unsigned long bootmap_size, start_pfn, max_low_pfn;
-
 	/*
 	 * partially used pages are not usable - thus
 	 * we are rounding upwards:
 	 */
-	start_pfn = PFN_UP(init_pg_tables_end);
+	min_low_pfn = PFN_UP(init_pg_tables_end);
 
 	find_max_pfn();
 

@@ -1040,10 +1038,19 @@ static unsigned long __init setup_memory
 #endif
 	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
 			pages_to_mb(max_low_pfn));
+
+	setup_bootmem_allocator();
+	return max_low_pfn;
+}
+#endif /* !CONFIG_DISCONTIGMEM */
+
+void __init setup_bootmem_allocator(void)
+{
+	unsigned long bootmap_size;
 	/*
 	 * Initialize the boot-time allocator (with low memory only):
 	 */
-	bootmap_size = init_bootmem(start_pfn, max_low_pfn);
+	bootmap_size = init_bootmem(min_low_pfn, max_low_pfn);
 
 	register_bootmem_low_pages(max_low_pfn);
 

@@ -1053,7 +1060,7 @@ static unsigned long __init setup_memory
 	 * the (very unlikely) case of us accidentally initializing the
 	 * bootmem allocator with an invalid RAM area.
 	 */
-	reserve_bootmem(HIGH_MEMORY, (PFN_PHYS(start_pfn) +
+	reserve_bootmem(HIGH_MEMORY, (PFN_PHYS(min_low_pfn) +
 			 bootmap_size + PAGE_SIZE-1) - (HIGH_MEMORY));
 
 	/*

@@ -1110,11 +1117,7 @@ static unsigned long __init setup_memory
 		}
 	}
 #endif
-	return max_low_pfn;
 }
-#else
-extern unsigned long setup_memory(void);
-#endif /* !CONFIG_DISCONTIGMEM */
 
 /*
  * Request address space for all standard RAM and ROM resources

139 lines not on this page · raw message

diff -puN arch/i386/kernel/setup.c~2_7_060_refactor_setup_memory_i386-cleanup arch/i386/kernel/setup.c
@@ -1014,36 +1014,6 @@ static void __init reserve_ebda_region(v
 		reserve_bootmem(addr, PAGE_SIZE);	
 }
 
-#ifndef CONFIG_DISCONTIGMEM
-void __init setup_bootmem_allocator(void);
-static unsigned long __init setup_memory(void)
-{
-	/*
-	 * partially used pages are not usable - thus
-	 * we are rounding upwards:
-	 */
-	min_low_pfn = PFN_UP(init_pg_tables_end);
-
-	find_max_pfn();
-
-	max_low_pfn = find_max_low_pfn();
-
-#ifdef CONFIG_HIGHMEM
-	highstart_pfn = highend_pfn = max_pfn;
-	if (max_pfn > max_low_pfn) {
-		highstart_pfn = max_low_pfn;
-	}
-	printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
-		pages_to_mb(highend_pfn - highstart_pfn));
-#endif
-	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
-			pages_to_mb(max_low_pfn));
-
-	setup_bootmem_allocator();
-	return max_low_pfn;
-}
-#endif /* !CONFIG_DISCONTIGMEM */
-
 void __init setup_bootmem_allocator(void)
 {
 	unsigned long bootmap_size;

@@ -1119,6 +1089,35 @@ void __init setup_bootmem_allocator(void
 #endif
 }
 
+#ifndef CONFIG_DISCONTIGMEM
+static unsigned long __init setup_memory(void)
+{
+	/*
+	 * partially used pages are not usable - thus
+	 * we are rounding upwards:
+	 */
+	min_low_pfn = PFN_UP(init_pg_tables_end);
+
+	find_max_pfn();
+
+	max_low_pfn = find_max_low_pfn();
+
+#ifdef CONFIG_HIGHMEM
+	highstart_pfn = highend_pfn = max_pfn;
+	if (max_pfn > max_low_pfn) {
+		highstart_pfn = max_low_pfn;
+	}
+	printk(KERN_NOTICE "%ldMB HIGHMEM available.\n",
+		pages_to_mb(highend_pfn - highstart_pfn));
+#endif
+	printk(KERN_NOTICE "%ldMB LOWMEM available.\n",
+			pages_to_mb(max_low_pfn));
+
+	setup_bootmem_allocator();
+	return max_low_pfn;
+}
+#endif /* !CONFIG_DISCONTIGMEM */
+
 /*
  * Request address space for all standard RAM and ROM resources
  * and also for regions reported as reserved by the e820.

diff -upN reference/arch/i386/mm/discontig.c current/arch/i386/mm/discontig.c
@@ -81,6 +81,9 @@ unsigned long node_remap_offset[MAX_NUMN
 void *node_remap_start_vaddr[MAX_NUMNODES];
 void set_pmd_pfn(unsigned long vaddr, unsigned long pfn, pgprot_t flags);
 
+void *node_remap_end_vaddr[MAX_NUMNODES];
+void *node_remap_alloc_vaddr[MAX_NUMNODES];
+
 /*
  * FLAT - support for basic PC memory model with discontig enabled, essentially
  *        a single node with all available processors in it with a flat

@@ -136,13 +139,36 @@ static void __init allocate_pgdat(int ni
 	}
 }
 
+void *alloc_remap(int nid, unsigned long size)
+{
+	void *allocation = node_remap_alloc_vaddr[nid];
+
+	printk(KERN_WARNING "APW: alloc_remap(%d, %08lx)\n", nid, size);
+
+	size = ALIGN(size, L1_CACHE_BYTES);
+
+	if (!allocation)
+	return 0;
+	if ((allocation + size) >= node_remap_end_vaddr[nid])
+		return 0;
+
+	node_remap_alloc_vaddr[nid] += size;
+
+	memset(allocation, 0, size);
+
+	printk(KERN_WARNING "APW: alloc_remap(%d, %08lx) = %p\n", nid, size,
+			allocation);
+
+	return allocation;
+}
+
 void __init remap_numa_kva(void)
 {
 	void *vaddr;
 	unsigned long pfn;
 	int node;
 
-	for (node = 1; node < numnodes; ++node) {
+	for (node = 0; node < numnodes; ++node) {
 		for (pfn=0; pfn < node_remap_size[node]; pfn += PTRS_PER_PTE) {
 			vaddr = node_remap_start_vaddr[node]+(pfn<<PAGE_SHIFT);
 			set_pmd_pfn((ulong) vaddr, 

@@ -152,15 +178,21 @@ void __init remap_numa_kva(void)
 	}
 }
 
+/* APW/XXX: not here .. */
+unsigned long zone_bitmap_calculate(unsigned long nr_pages);
 static unsigned long calculate_numa_remap_pages(void)
 {
 	int nid;
 	unsigned long size, reserve_pages = 0;
 
-	for (nid = 1; nid < numnodes; nid++) {
+	for (nid = 0; nid < numnodes; nid++) {
 		/* calculate the size of the mem_map needed in bytes */
 		size = (node_end_pfn[nid] - node_start_pfn[nid] + 1) 
 			* sizeof(struct page) + sizeof(pg_data_t);
+
+		/* Allow for the bitmaps. */
+		size += zone_bitmap_calculate(node_end_pfn[nid] - node_start_pfn[nid] + 1);
+
 		/* convert size to large (pmd size) pages, rounding up */
 		size = (size + LARGE_PAGE_BYTES - 1) / LARGE_PAGE_BYTES;
 		/* now the roundup is correct, convert to PAGE_SIZE pages */

@@ -168,8 +200,8 @@ static unsigned long calculate_numa_rema
 		printk("Reserving %ld pages of KVA for lmem_map of node %d\n",
 				size, nid);
 		node_remap_size[nid] = size;
-		reserve_pages += size;
 		node_remap_offset[nid] = reserve_pages;
+		reserve_pages += size;
 		printk("Shrinking node %d from %ld pages to %ld pages\n",
 			nid, node_end_pfn[nid], node_end_pfn[nid] - size);
 		node_end_pfn[nid] -= size;

@@ -236,12 +268,18 @@ unsigned long __init setup_memory(void)
 			(ulong) pfn_to_kaddr(max_low_pfn));
 	for (nid = 0; nid < numnodes; nid++) {
 		node_remap_start_vaddr[nid] = pfn_to_kaddr(
-			(highstart_pfn + reserve_pages) - node_remap_offset[nid]);
+			highstart_pfn + node_remap_offset[nid]);
+		/* Init the node remap allocator */
+		node_remap_end_vaddr[nid] = node_remap_start_vaddr[nid] +
+			(node_remap_size[nid] * PAGE_SIZE);
+		node_remap_alloc_vaddr[nid] = node_remap_start_vaddr[nid] +
+			ALIGN(sizeof(pg_data_t), PAGE_SIZE);
+
 		allocate_pgdat(nid);
 		printk ("node %d will remap to vaddr %08lx - %08lx\n", nid,
 			(ulong) node_remap_start_vaddr[nid],
-			(ulong) pfn_to_kaddr(highstart_pfn + reserve_pages
-			    - node_remap_offset[nid] + node_remap_size[nid]));
+			(ulong) pfn_to_kaddr(highstart_pfn 
+			    + node_remap_offset[nid] + node_remap_size[nid]));
 	}
 	printk("High memory starts at vaddr %08lx\n",
 			(ulong) pfn_to_kaddr(highstart_pfn));

@@ -307,6 +345,10 @@ void __init zone_sizes_init(void)
 		 * normal bootmem allocator, but other nodes come from the
 		 * remapped KVA area - mbligh
 		 */
+			free_area_init_node(nid, NODE_DATA(nid),
+					zones_size, start, zholes_size);
+
+#if 0
 		if (!nid)
 			free_area_init_node(nid, NODE_DATA(nid),
 					zones_size, start, zholes_size);

@@ -319,6 +361,7 @@ void __init zone_sizes_init(void)
 			free_area_init_node(nid, NODE_DATA(nid), zones_size,
 				start, zholes_size);
 		}
+#endif
 	}
 	return;
 }

diff -upN reference/include/asm-i386/mmzone.h current/include/asm-i386/mmzone.h
@@ -16,6 +16,8 @@
 	#else	/* summit or generic arch */
 		#include <asm/srat.h>
 	#endif
+	#define HAVE_ARCH_ALLOC_REMAP	1
+
 #else /* !CONFIG_NUMA */
 	#define get_memcfg_numa get_memcfg_numa_flat
 	#define get_zholes_size(n) (0)

diff -upN reference/mm/page_alloc.c current/mm/page_alloc.c
@@ -94,6 +94,9 @@ static void bad_page(const char *functio
 	page->mapping = NULL;
 }
 
+/* APW/XXX: not here. */
+void *alloc_remap(int nid, unsigned long size);
+
 #ifndef CONFIG_HUGETLB_PAGE
 #define prep_compound_page(page, order) do { } while (0)
 #define destroy_compound_page(page, order) do { } while (0)

@@ -1442,11 +1445,23 @@ unsigned long pages_to_bitmap_size(unsig
 	return bitmap_size;
 }
 
+unsigned long zone_bitmap_calculate(unsigned long nr_pages)
+{
+	unsigned long overall_size = 0;
+	int order;
+
+	for (order = 0; order < MAX_ORDER - 1; order++)
+		overall_size += pages_to_bitmap_size(order, nr_pages);
+	
+	return overall_size;
+}
+
 void zone_init_free_lists(struct pglist_data *pgdat, struct zone *zone, unsigned long size)
 {
 	int order;
 	for (order = 0; ; order++) {
 		unsigned long bitmap_size;
+		unsigned long *map;
 
 		INIT_LIST_HEAD(&zone->free_area[order].free_list);
 		if (order == MAX_ORDER-1) {

@@ -1455,8 +1470,15 @@ void zone_init_free_lists(struct pglist_
 		}
 
 		bitmap_size = pages_to_bitmap_size(order, size);
-		zone->free_area[order].map =
-		  (unsigned long *) alloc_bootmem_node(pgdat, bitmap_size);
+
+#ifdef HAVE_ARCH_ALLOC_REMAP
+		map = (unsigned long *) alloc_remap(pgdat->node_id,
+			bitmap_size);
+		if (!map) 
+#endif
+			map = (unsigned long *) alloc_bootmem_node(pgdat,
+				bitmap_size);
+		zone->free_area[order].map = map;
 	}
 }
 

@@ -1581,9 +1603,16 @@ static void __init free_area_init_core(s
 void __init node_alloc_mem_map(struct pglist_data *pgdat)
 {
 	unsigned long size;
+	void *map;
 
 	size = (pgdat->node_spanned_pages + 1) * sizeof(struct page);
-	pgdat->node_mem_map = alloc_bootmem_node(pgdat, size);
+
+#ifdef HAVE_ARCH_ALLOC_REMAP
+	map = (unsigned long *) alloc_remap(pgdat->node_id, size);
+	if (!map)
+#endif
+		map = alloc_bootmem_node(pgdat, size);
+	pgdat->node_mem_map = map;
 #ifndef CONFIG_DISCONTIGMEM
 	mem_map = contig_page_data.node_mem_map;
 #endif

diff -puN arch/i386/mm/discontig.c~3_7_080_alloc_remap_i386-whitespace arch/i386/mm/discontig.c
@@ -278,7 +278,7 @@ unsigned long __init setup_memory(void)
 		allocate_pgdat(nid);
 		printk ("node %d will remap to vaddr %08lx - %08lx\n", nid,
 			(ulong) node_remap_start_vaddr[nid],
-			(ulong) pfn_to_kaddr(highstart_pfn 
+			(ulong) pfn_to_kaddr(highstart_pfn
 			    + node_remap_offset[nid] + node_remap_size[nid]));
 	}
 	printk("High memory starts at vaddr %08lx\n",

diff -puN mm/page_alloc.c~3_7_080_alloc_remap_i386-whitespace mm/page_alloc.c
@@ -1474,7 +1474,7 @@ void zone_init_free_lists(struct pglist_
 #ifdef HAVE_ARCH_ALLOC_REMAP
 		map = (unsigned long *) alloc_remap(pgdat->node_id,
 			bitmap_size);
-		if (!map) 
+		if (!map)
 #endif
 			map = (unsigned long *) alloc_bootmem_node(pgdat,
 				bitmap_size);

diff -puN include/linux/bootmem.h~3_7_080_alloc_remap_i386-removeifdefs include/linux/bootmem.h
@@ -67,6 +67,15 @@ extern void * __init __alloc_bootmem_nod
 	__alloc_bootmem_node((pgdat), (x), PAGE_SIZE, 0)
 #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
 
+#ifdef HAVE_ARCH_ALLOC_REMAP
+extern void *alloc_remap(int nid, unsigned long size);
+#else
+static inline void *alloc_remap(int nid, unsigned long size)
+{
+	return NULL;
+}
+#endif
+
 extern unsigned long __initdata nr_kernel_pages;
 extern unsigned long __initdata nr_all_pages;
 

diff -upN reference/include/linux/mm.h current/include/linux/mm.h
@@ -376,16 +376,41 @@ static inline void put_page(struct page 
  * We'll have up to (MAX_NUMNODES * MAX_NR_ZONES) zones total,
  * so we use (MAX_NODES_SHIFT + MAX_ZONES_SHIFT) here to get enough bits.
  */
-#define NODEZONE_SHIFT (sizeof(page_flags_t)*8 - MAX_NODES_SHIFT - MAX_ZONES_SHIFT)
-#define NODEZONE(node, zone)	((node << ZONES_SHIFT) | zone)
+
+#define FLAGS_SHIFT	(sizeof(page_flags_t)*8)
+
+/* 32bit: NODE:ZONE */
+#define PGFLAGS_NODES_SHIFT	(FLAGS_SHIFT - NODES_SHIFT)
+#define PGFLAGS_ZONES_SHIFT	(PGFLAGS_NODES_SHIFT - ZONES_SHIFT)
+
+#define ZONETABLE_SHIFT		(NODES_SHIFT + ZONES_SHIFT)
+#define PGFLAGS_ZONETABLE_SHIFT	(FLAGS_SHIFT - ZONETABLE_SHIFT)
+
+#if NODES_SHIFT+ZONES_SHIFT > FLAGS_TOTAL_SHIFT
+#error NODES_SHIFT+ZONES_SHIFT > FLAGS_TOTAL_SHIFT
+#endif
+
+#define NODEZONE(node, zone)		((node << ZONES_SHIFT) | zone)
+
+#define ZONES_MASK		(~((~0UL) << ZONES_SHIFT))
+#define NODES_MASK		(~((~0UL) << NODES_SHIFT))
+#define ZONETABLE_MASK		(~((~0UL) << ZONETABLE_SHIFT))
+
+#define PGFLAGS_MASK		(~((~0UL) << PGFLAGS_ZONETABLE_SHIFT)
 
 static inline unsigned long page_zonenum(struct page *page)
 {
-	return (page->flags >> NODEZONE_SHIFT) & (~(~0UL << ZONES_SHIFT));
+	if (FLAGS_SHIFT == (PGFLAGS_ZONES_SHIFT + ZONES_SHIFT))
+ 		return (page->flags >> PGFLAGS_ZONES_SHIFT);
+ 	else
+ 		return (page->flags >> PGFLAGS_ZONES_SHIFT) & ZONES_MASK;
 }
 static inline unsigned long page_to_nid(struct page *page)
 {
-	return (page->flags >> (NODEZONE_SHIFT + ZONES_SHIFT));
+	if (FLAGS_SHIFT == (PGFLAGS_NODES_SHIFT + NODES_SHIFT))
+		return (page->flags >> PGFLAGS_NODES_SHIFT);
+	else
+		return (page->flags >> PGFLAGS_NODES_SHIFT) & NODES_MASK;
 }
 
 struct zone;

@@ -393,13 +418,17 @@ extern struct zone *zone_table[];
 
 static inline struct zone *page_zone(struct page *page)
 {
-	return zone_table[page->flags >> NODEZONE_SHIFT];
+	if (FLAGS_SHIFT == (PGFLAGS_ZONETABLE_SHIFT + ZONETABLE_SHIFT))
+		return zone_table[page->flags >> PGFLAGS_ZONETABLE_SHIFT];
+	else
+		return zone_table[page->flags >> PGFLAGS_ZONETABLE_SHIFT &
+			ZONETABLE_MASK];
 }
 
 static inline void set_page_zone(struct page *page, unsigned long nodezone_num)
 {
-	page->flags &= ~(~0UL << NODEZONE_SHIFT);
-	page->flags |= nodezone_num << NODEZONE_SHIFT;
+	page->flags &= PGFLAGS_MASK;
+	page->flags |= nodezone_num << PGFLAGS_ZONETABLE_SHIFT;
 }
 
 #ifndef CONFIG_DISCONTIGMEM

diff -upN reference/include/linux/mmzone.h current/include/linux/mmzone.h
@@ -389,27 +389,17 @@ extern struct pglist_data contig_page_da
  * with 32 bit page->flags field, we reserve 8 bits for node/zone info.
  * there are 3 zones (2 bits) and this leaves 8-2=6 bits for nodes.
  */
-#define MAX_NODES_SHIFT		6
+#define FLAGS_TOTAL_SHIFT	8
+
 #elif BITS_PER_LONG == 64
 /*
  * with 64 bit flags field, there's plenty of room.
  */
-#define MAX_NODES_SHIFT		10
+#define FLAGS_TOTAL_SHIFT	12
 #endif
 
 #endif /* !CONFIG_DISCONTIGMEM */
 
-#if NODES_SHIFT > MAX_NODES_SHIFT
-#error NODES_SHIFT > MAX_NODES_SHIFT
-#endif
-
-/* There are currently 3 zones: DMA, Normal & Highmem, thus we need 2 bits */
-#define MAX_ZONES_SHIFT		2
-
-#if ZONES_SHIFT > MAX_ZONES_SHIFT
-#error ZONES_SHIFT > MAX_ZONES_SHIFT
-#endif
-
 extern DECLARE_BITMAP(node_online_map, MAX_NUMNODES);
 
 #if defined(CONFIG_DISCONTIGMEM) || defined(CONFIG_NUMA)

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/include/linux/mm.h current/include/linux/mm.h
@@ -379,24 +379,76 @@ static inline void put_page(struct page 
 
 #define FLAGS_SHIFT	(sizeof(page_flags_t)*8)
 
-/* 32bit: NODE:ZONE */
+/*
+ * CONFIG_SPARSEMEM:
+ *   If there is room for SECTIONS, NODES AND ZONES then:
+ *     NODE:ZONE:SECTION
+ *   else:
+ *     SECTION:ZONE
+ *
+ * Otherwise:
+ *   NODE:ZONE
+ */
+#ifdef CONFIG_SPARSEMEM
+
+#if FLAGS_TOTAL_SHIFT >= SECTIONS_SHIFT + NODES_SHIFT + ZONES_SHIFT
+
+/* NODE:ZONE:SECTION */
 #define PGFLAGS_NODES_SHIFT	(FLAGS_SHIFT - NODES_SHIFT)
 #define PGFLAGS_ZONES_SHIFT	(PGFLAGS_NODES_SHIFT - ZONES_SHIFT)
+#define PGFLAGS_SECTIONS_SHIFT	(PGFLAGS_ZONES_SHIFT - SECTIONS_SHIFT)
+
+#define FLAGS_USED_SHIFT	(NODES_SHIFT + ZONES_SHIFT + SECTIONS_SHIFT)
 
 #define ZONETABLE_SHIFT		(NODES_SHIFT + ZONES_SHIFT)
 #define PGFLAGS_ZONETABLE_SHIFT	(FLAGS_SHIFT - ZONETABLE_SHIFT)
 
-#if NODES_SHIFT+ZONES_SHIFT > FLAGS_TOTAL_SHIFT
-#error NODES_SHIFT+ZONES_SHIFT > FLAGS_TOTAL_SHIFT
+#define ZONETABLE(section, node, zone) \
+			((node << ZONES_SHIFT) | zone)
+
+#else
+
+/* SECTION:ZONE */
+#define PGFLAGS_SECTIONS_SHIFT	(FLAGS_SHIFT - SECTIONS_SHIFT)
+#define PGFLAGS_ZONES_SHIFT	(PGFLAGS_SECTIONS_SHIFT - ZONES_SHIFT)
+
+#define FLAGS_USED_SHIFT	(SECTIONS_SHIFT + ZONES_SHIFT)
+
+#define ZONETABLE_SHIFT		(SECTIONS_SHIFT + ZONES_SHIFT)
+#define PGFLAGS_ZONETABLE_SHIFT	(FLAGS_SHIFT - ZONETABLE_SHIFT)
+
+#define ZONETABLE(section, node, zone) \
+			((section << ZONES_SHIFT) | zone)
+
+#endif
+
+#else /* !CONFIG_SPARSEMEM */
+
+/* NODE:ZONE */
+#define PGFLAGS_NODES_SHIFT	(FLAGS_SHIFT - NODES_SHIFT)
+#define PGFLAGS_ZONES_SHIFT	(PGFLAGS_NODES_SHIFT - ZONES_SHIFT)
+
+#define ZONETABLE_SHIFT		(NODES_SHIFT + ZONES_SHIFT)
+#define PGFLAGS_ZONETABLE_SHIFT	(FLAGS_SHIFT - ZONETABLE_SHIFT)
+
+#define FLAGS_USED_SHIFT	(NODES_SHIFT + ZONES_SHIFT)
+
+#endif /* !CONFIG_SPARSEMEM */
+
+#if FLAGS_USED_SHIFT > FLAGS_TOTAL_SHIFT
+#error SECTIONS_SHIFT+NODES_SHIFT+ZONES_SHIFT > FLAGS_TOTAL_SHIFT
 #endif
 
 #define NODEZONE(node, zone)		((node << ZONES_SHIFT) | zone)
 
 #define ZONES_MASK		(~((~0UL) << ZONES_SHIFT))
 #define NODES_MASK		(~((~0UL) << NODES_SHIFT))
+#define SECTIONS_MASK		(~((~0UL) << SECTIONS_SHIFT))
 #define ZONETABLE_MASK		(~((~0UL) << ZONETABLE_SHIFT))
 
-#define PGFLAGS_MASK		(~((~0UL) << PGFLAGS_ZONETABLE_SHIFT)
+#define ZONETABLE_SIZE  	(1 << ZONETABLE_SHIFT)
+
+#define PGFLAGS_MASK		(~((~0UL) << PGFLAGS_ZONETABLE_SHIFT))
 
 static inline unsigned long page_zonenum(struct page *page)
 {

@@ -405,13 +457,34 @@ static inline unsigned long page_zonenum
  	else
  		return (page->flags >> PGFLAGS_ZONES_SHIFT) & ZONES_MASK;
 }
+#ifdef PGFLAGS_NODES_SHIFT
 static inline unsigned long page_to_nid(struct page *page)
 {
+#if NODES_SHIFT == 0
+	return 0;
+#else 
 	if (FLAGS_SHIFT == (PGFLAGS_NODES_SHIFT + NODES_SHIFT))
 		return (page->flags >> PGFLAGS_NODES_SHIFT);
 	else
 		return (page->flags >> PGFLAGS_NODES_SHIFT) & NODES_MASK;
+#endif
 }
+#else
+static inline struct zone *page_zone(struct page *page);
+static inline unsigned long page_to_nid(struct page *page)
+{
+	return page_zone(page)->zone_pgdat->node_id;
+}
+#endif
+#ifdef PGFLAGS_SECTIONS_SHIFT
+static inline unsigned long page_to_section(struct page *page)
+{
+	if (FLAGS_SHIFT == (PGFLAGS_SECTIONS_SHIFT + SECTIONS_SHIFT))
+ 		return (page->flags >> PGFLAGS_SECTIONS_SHIFT);
+ 	else
+ 		return (page->flags >> PGFLAGS_SECTIONS_SHIFT) & SECTIONS_MASK;
+}
+#endif
 
 struct zone;
 extern struct zone *zone_table[];

@@ -425,13 +498,27 @@ static inline struct zone *page_zone(str
 			ZONETABLE_MASK];
 }
 
-static inline void set_page_zone(struct page *page, unsigned long nodezone_num)
+static inline void set_page_zone(struct page *page, unsigned long zone)
+{
+	page->flags &= ~(ZONES_MASK << PGFLAGS_ZONES_SHIFT);
+	page->flags |= zone << PGFLAGS_ZONES_SHIFT;
+}
+static inline void set_page_node(struct page *page, unsigned long node)
 {
-	page->flags &= PGFLAGS_MASK;
-	page->flags |= nodezone_num << PGFLAGS_ZONETABLE_SHIFT;
+#if defined(PGFLAGS_NODES_SHIFT) && NODES_SHIFT != 0
+	page->flags &= ~(NODES_MASK << PGFLAGS_NODES_SHIFT);
+	page->flags |= node << PGFLAGS_NODES_SHIFT;
+#endif
+}
+static inline void set_page_section(struct page *page, unsigned long section)
+{
+#ifdef PGFLAGS_SECTIONS_SHIFT
+	page->flags &= ~(SECTIONS_MASK << PGFLAGS_SECTIONS_SHIFT);
+	page->flags |= section << PGFLAGS_SECTIONS_SHIFT;
+#endif
 }
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 /* The array of struct pages - for discontigmem use pgdat->lmem_map */
 extern struct page *mem_map;
 #endif

165 lines not on this page · raw message

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/include/linux/numa.h current/include/linux/numa.h
@@ -3,7 +3,7 @@
 
 #include <linux/config.h>
 
-#ifdef CONFIG_DISCONTIGMEM
+#ifndef CONFIG_FLATMEM
 #include <asm/numnodes.h>
 #endif
 

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/init/main.c current/init/main.c
@@ -480,6 +480,7 @@ asmlinkage void __init start_kernel(void
 {
 	char * command_line;
 	extern struct kernel_param __start___param[], __stop___param[];
+
 /*
  * Interrupts are still disabled. Do necessary setups, then
  * enable them

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/mm/bootmem.c current/mm/bootmem.c
@@ -255,6 +255,7 @@ found:
 static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
 {
 	struct page *page;
+	unsigned long pfn;
 	bootmem_data_t *bdata = pgdat->bdata;
 	unsigned long i, count, total = 0;
 	unsigned long idx;

@@ -265,15 +266,26 @@ static unsigned long __init free_all_boo
 
 	count = 0;
 	/* first extant page of the node */
-	page = virt_to_page(phys_to_virt(bdata->node_boot_start));
+	pfn = bdata->node_boot_start >> PAGE_SHIFT;
 	idx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT);
 	map = bdata->node_bootmem_map;
 	/* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
 	if (bdata->node_boot_start == 0 ||
 	    ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG))
 		gofast = 1;
+	page = pfn_to_page(pfn);
 	for (i = 0; i < idx; ) {
 		unsigned long v = ~map[i / BITS_PER_LONG];
+
+		/*
+		 * Makes use of the guarentee that *_mem_map will be
+		 * contigious in sections aligned at MAX_ORDER.
+		 * APW/XXX: we are making an assumption that our node_boot_start
+		 * is aligned to BITS_PER_LONG ... is this valid/enforced.
+		 */
+		if ((pfn & ((1 << MAX_ORDER) - 1)) == 0)
+			page = pfn_to_page(pfn);
+
 		if (gofast && v == ~0UL) {
 			int j;
 

@@ -302,6 +314,7 @@ static unsigned long __init free_all_boo
 			i+=BITS_PER_LONG;
 			page += BITS_PER_LONG;
 		}
+		pfn += BITS_PER_LONG;
 	}
 	total += count;
 

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/mm/memory.c current/mm/memory.c
@@ -56,7 +56,7 @@
 #include <linux/swapops.h>
 #include <linux/elf.h>
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 /* use the per-pgdat data instead for discontigmem - mbligh */
 unsigned long max_mapnr;
 struct page *mem_map;

137 lines not on this page · raw message

163 lines not on this page · raw message

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/arch/i386/kernel/numaq.c current/arch/i386/kernel/numaq.c
@@ -60,6 +60,11 @@ static void __init smp_dump_qct(void)
 				eq->hi_shrd_mem_start - eq->priv_mem_size);
 			node_end_pfn[node] = MB_TO_PAGES(
 				eq->hi_shrd_mem_start + eq->hi_shrd_mem_size);
+#ifdef CONFIG_SPARSEMEM
+			sparse_add(node, node_start_pfn[node],
+				node_end_pfn[node]);
+#endif
+
 		}
 	}
 }

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/arch/i386/kernel/setup.c current/arch/i386/kernel/setup.c
@@ -39,6 +39,7 @@
 #include <linux/efi.h>
 #include <linux/init.h>
 #include <linux/edd.h>
+#include <linux/mmzone.h>
 #include <video/edid.h>
 #include <asm/e820.h>
 #include <asm/mpspec.h>

@@ -1014,7 +1015,7 @@ static void __init reserve_ebda_region(v
 		reserve_bootmem(addr, PAGE_SIZE);	
 }
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 void __init setup_bootmem_allocator(void);
 static unsigned long __init setup_memory(void)
 {

@@ -1042,7 +1043,9 @@ static unsigned long __init setup_memory
 	setup_bootmem_allocator();
 	return max_low_pfn;
 }
-#endif /* !CONFIG_DISCONTIGMEM */
+#else
+unsigned long __init setup_memory(void);
+#endif /* CONFIG_FLATMEM */
 
 void __init setup_bootmem_allocator(void)
 {

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/arch/i386/kernel/srat.c current/arch/i386/kernel/srat.c
@@ -261,6 +261,11 @@ static int __init acpi20_parse_srat(stru
 		       j, node_memory_chunk[j].nid,
 		       node_memory_chunk[j].start_pfn,
 		       node_memory_chunk[j].end_pfn);
+#ifdef CONFIG_SPARSEMEM
+		 sparse_add(node_memory_chunk[j].nid,
+				 node_memory_chunk[j].start_pfn,
+				 node_memory_chunk[j].end_pfn);
+#endif /* CONFIG_SPARSEMEM */
 	}
  
 	/*calculate node_start_pfn/node_end_pfn arrays*/

165 lines not on this page · raw message

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/arch/i386/mm/init.c current/arch/i386/mm/init.c
@@ -274,7 +274,7 @@ void __init one_highpage_init(struct pag
 		SetPageReserved(page);
 }
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 void __init set_highmem_pages_init(int bad_ppro) 
 {
 	int pfn;

@@ -284,7 +284,7 @@ void __init set_highmem_pages_init(int b
 }
 #else
 extern void set_highmem_pages_init(int);
-#endif /* !CONFIG_DISCONTIGMEM */
+#endif /* CONFIG_FLATMEM */
 
 #else
 #define kmap_init() do { } while (0)

@@ -295,7 +295,7 @@ extern void set_highmem_pages_init(int);
 unsigned long long __PAGE_KERNEL = _PAGE_KERNEL;
 unsigned long long __PAGE_KERNEL_EXEC = _PAGE_KERNEL_EXEC;
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 #define remap_numa_kva() do {} while (0)
 #else
 extern void __init remap_numa_kva(void);

@@ -388,7 +388,7 @@ void zap_low_mappings (void)
 	flush_tlb_all();
 }
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 void __init zone_sizes_init(void)
 {
 	unsigned long zones_size[MAX_NR_ZONES] = {0, 0, 0};

@@ -411,7 +411,7 @@ void __init zone_sizes_init(void)
 }
 #else
 extern void zone_sizes_init(void);
-#endif /* !CONFIG_DISCONTIGMEM */
+#endif /* CONFIG_FLATMEM */
 
 static int disable_nx __initdata = 0;
 u64 __supported_pte_mask = ~_PAGE_NX;

@@ -516,6 +516,9 @@ void __init paging_init(void)
 	__flush_tlb_all();
 
 	kmap_init();
+#ifdef CONFIG_SPARSEMEM
+	sparse_allocate();
+#endif
 	zone_sizes_init();
 }
 

@@ -545,7 +548,7 @@ void __init test_wp_bit(void)
 	}
 }
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 static void __init set_max_mapnr_init(void)
 {
 #ifdef CONFIG_HIGHMEM

@@ -559,7 +562,7 @@ static void __init set_max_mapnr_init(vo
 #else
 #define __free_all_bootmem() free_all_bootmem_node(NODE_DATA(0))
 extern void set_max_mapnr_init(void);
-#endif /* !CONFIG_DISCONTIGMEM */
+#endif /* CONFIG_FLATMEM */
 
 static struct kcore_list kcore_mem, kcore_vmalloc; 
 

@@ -570,7 +573,7 @@ void __init mem_init(void)
 	int tmp;
 	int bad_ppro;
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 	if (!mem_map)
 		BUG();
 #endif

149 lines not on this page · raw message

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/include/asm-i386/page.h current/include/asm-i386/page.h
@@ -133,11 +133,11 @@ extern int sysctl_legacy_va_layout;
 #define __pa(x)			((unsigned long)(x)-PAGE_OFFSET)
 #define __va(x)			((void *)((unsigned long)(x)+PAGE_OFFSET))
 #define pfn_to_kaddr(pfn)      __va((pfn) << PAGE_SHIFT)
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 #define pfn_to_page(pfn)	(mem_map + (pfn))
 #define page_to_pfn(page)	((unsigned long)((page) - mem_map))
 #define pfn_valid(pfn)		((pfn) < max_mapnr)
-#endif /* !CONFIG_DISCONTIGMEM */
+#endif /* CONFIG_FLATMEM */
 #define virt_to_page(kaddr)	pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
 
 #define virt_addr_valid(kaddr)	pfn_valid(__pa(kaddr) >> PAGE_SHIFT)

diff -X /home/apw/brief/lib/vdiff.excl -rupN reference/include/asm-i386/pgtable.h current/include/asm-i386/pgtable.h
@@ -400,9 +400,9 @@ extern pte_t *lookup_address(unsigned lo
 
 #endif /* !__ASSEMBLY__ */
 
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 #define kern_addr_valid(addr)	(1)
-#endif /* !CONFIG_DISCONTIGMEM */
+#endif /* CONFIG_FLATMEM */
 
 #define io_remap_page_range remap_page_range
 

diff -upN reference/arch/ppc64/mm/init.c current/arch/ppc64/mm/init.c
@@ -597,7 +597,7 @@ EXPORT_SYMBOL(page_is_ram);
  * Initialize the bootmem system and give it all the memory we
  * have available.
  */
-#ifndef CONFIG_DISCONTIGMEM
+#ifdef CONFIG_FLATMEM
 void __init do_init_bootmem(void)
 {
 	unsigned long i;

@@ -695,7 +695,7 @@ module_init(setup_kcore);
 
 void __init mem_init(void)
 {
-#ifdef CONFIG_DISCONTIGMEM
+#if defined(CONFIG_DISCONTIGMEM) || defined(CONFIG_SPARSEMEM)
 	int nid;
 #endif
 	pg_data_t *pgdat;

@@ -706,7 +706,7 @@ void __init mem_init(void)
 	num_physpages = max_low_pfn;	/* RAM is assumed contiguous */
 	high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
 
-#ifdef CONFIG_DISCONTIGMEM
+#if defined(CONFIG_DISCONTIGMEM) || defined(CONFIG_SPARSEMEM)
         for (nid = 0; nid < numnodes; nid++) {
 		if (NODE_DATA(nid)->node_spanned_pages != 0) {
 			printk("freeing bootmem node %x\n", nid);

@@ -721,7 +721,7 @@ void __init mem_init(void)
 
 	for_each_pgdat(pgdat) {
 		for (i = 0; i < pgdat->node_spanned_pages; i++) {
-			page = pgdat->node_mem_map + i;
+			page = pfn_to_page(i);
 			if (PageReserved(page))
 				reservedpages++;
 		}

diff -upN reference/arch/ppc64/mm/numa.c current/arch/ppc64/mm/numa.c
@@ -304,9 +304,13 @@ new_range:
 				size / PAGE_SIZE;
 		}
 
+		/* XXX: think this is discontig ... */
 		for (i = start ; i < (start+size); i += MEMORY_INCREMENT)
 			numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] =
 				numa_domain;
+#ifdef CONFIG_SPARSEMEM
+		sparse_add(numa_domain, start, start + size);
+#endif
 
 		ranges--;
 		if (ranges)

@@ -346,10 +350,15 @@ static void __init setup_nonnuma(void)
 	init_node_data[0].node_start_pfn = 0;
 	init_node_data[0].node_spanned_pages = lmb_end_of_DRAM() / PAGE_SIZE;
 
+	/* APW: this is discontig? */
 	for (i = 0 ; i < top_of_ram; i += MEMORY_INCREMENT)
 		numa_memory_lookup_table[i >> MEMORY_INCREMENT_SHIFT] = 0;
 
 	node0_io_hole_size = top_of_ram - total_ram;
+
+#ifdef CONFIG_SPARSEMEM
+		sparse_add(0, 0, init_node_data[0].node_spanned_pages);
+#endif
 }
 
 static void __init dump_numa_topology(void)

@@ -567,6 +576,10 @@ void __init paging_init(void)
 	memset(zones_size, 0, sizeof(zones_size));
 	memset(zholes_size, 0, sizeof(zholes_size));
 
+#ifdef CONFIG_SPARSEMEM
+	sparse_allocate();
+#endif
+
 	for (nid = 0; nid < numnodes; nid++) {
 		unsigned long start_pfn;
 		unsigned long end_pfn;

diff -upN reference/include/asm-ppc64/mmzone.h current/include/asm-ppc64/mmzone.h
@@ -10,9 +10,13 @@
 #include <linux/config.h>
 #include <asm/smp.h>
 
-#ifdef CONFIG_DISCONTIGMEM
+#if defined(CONFIG_DISCONTIGMEM) || defined(CONFIG_SPARSEMEM)
 
 extern struct pglist_data *node_data[];
+/*
+ * Return a pointer to the node data for node n.
+ */
+#define NODE_DATA(nid)		(node_data[nid])
 
 /*
  * Following are specific to this numa platform.

@@ -27,6 +31,10 @@ extern int nr_cpus_in_node[];
 #define MEMORY_INCREMENT_SHIFT 24
 #define MEMORY_INCREMENT (1UL << MEMORY_INCREMENT_SHIFT)
 
+#endif /* !CONFIG_DISCONTIGMEM || !CONFIG_SPARSEMEM */
+
+#ifdef CONFIG_DISCONTIGMEM
+
 /* NUMA debugging, will not work on a DLPAR machine */
 #undef DEBUG_NUMA
 

@@ -49,11 +57,6 @@ static inline int pa_to_nid(unsigned lon
 
 #define pfn_to_nid(pfn)		pa_to_nid((pfn) << PAGE_SHIFT)
 
-/*
- * Return a pointer to the node data for node n.
- */
-#define NODE_DATA(nid)		(node_data[nid])
-
 #define node_localnr(pfn, nid)	((pfn) - NODE_DATA(nid)->node_start_pfn)
 
 /*

@@ -91,4 +94,29 @@ static inline int pa_to_nid(unsigned lon
 #define discontigmem_pfn_valid(pfn)		((pfn) < num_physpages)
 
 #endif /* CONFIG_DISCONTIGMEM */
+
+#ifdef CONFIG_SPARSEMEM
+
+/* generic non-linear memory support:
+ *
+ * 1) we will not split memory into more chunks than will fit into the
+ *    flags field of the struct page
+ */
+
+/*
+ * SECTION_SIZE_BITS            2^N: how big each section will be
+ * MAX_PHYSADDR_BITS            2^N: how much physical address space we have
+ * MAX_PHYSMEM_BITS             2^N: how much memory we can have in that space
+ */
+#define SECTION_SIZE_BITS       24
+#define MAX_PHYSADDR_BITS       38
+#define MAX_PHYSMEM_BITS        36
+
+#define pa_to_nid(pa)							\
+({									\
+	pfn_to_nid(pa >> PAGE_SHIFT);					\
+})
+
+#endif /* CONFIG_SPARSEMEM */
+
 #endif /* _ASM_MMZONE_H_ */

diff -upN reference/include/asm-ppc64/page.h current/include/asm-ppc64/page.h
@@ -222,7 +222,9 @@ extern int page_is_ram(unsigned long pfn
 #define page_to_pfn(page)	discontigmem_page_to_pfn(page)
 #define pfn_to_page(pfn)	discontigmem_pfn_to_page(pfn)
 #define pfn_valid(pfn)		discontigmem_pfn_valid(pfn)
-#else
+#endif
+/* XXX/APW: why is SPARSEMEM not here */
+#ifdef CONFIG_FLATMEM
 #define pfn_to_page(pfn)	(mem_map + (pfn))
 #define page_to_pfn(page)	((unsigned long)((page) - mem_map))
 #define pfn_valid(pfn)		((pfn) < max_mapnr)