pcp: zonequeues

Index: linux-2.6/include/linux/mmzone.h
===================================================================
@@ -53,14 +53,15 @@ struct per_cpu_pages {
 
 struct per_cpu_pageset {
 	struct per_cpu_pages pcp[2];	/* 0: hot.  1: cold */
-#ifdef CONFIG_NUMA
+};
+
+struct per_cpu_zone_stats {
 	unsigned long numa_hit;		/* allocated in intended node */
 	unsigned long numa_miss;	/* allocated in non intended node */
 	unsigned long numa_foreign;	/* was intended here, hit elsewhere */
 	unsigned long interleave_hit; 	/* interleaver prefered this zone */
 	unsigned long local_node;	/* allocation from local node */
 	unsigned long other_node;	/* allocation from other node */
-#endif
 } ____cacheline_aligned_in_smp;
 
 #define ZONE_DMA		0

@@ -113,16 +114,19 @@ struct zone {
 	unsigned long		free_pages;
 	unsigned long		pages_min, pages_low, pages_high;
 	/*
-	 * We don't know if the memory that we're going to allocate will be freeable
-	 * or/and it will be released eventually, so to avoid totally wasting several
-	 * GB of ram we must reserve some of the lower zone memory (otherwise we risk
-	 * to run OOM on the lower zones despite there's tons of freeable ram
-	 * on the higher zones). This array is recalculated at runtime if the
-	 * sysctl_lowmem_reserve_ratio sysctl changes.
+	 * We don't know if the memory that we're going to allocate will be
+	 * freeable or/and it will be released eventually, so to avoid totally
+	 * wasting several GB of ram we must reserve some of the lower zone
+	 * memory (otherwise we risk to run OOM on the lower zones despite
+	 * there's tons of freeable ram on the higher zones). This array is
+	 * recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl
+	 * changes.
 	 */
 	unsigned long		lowmem_reserve[MAX_NR_ZONES];
 
-	struct per_cpu_pageset	pageset[NR_CPUS];
+#ifdef CONFIG_NUMA
+	struct per_cpu_zone_stats stats[NR_CPUS];
+#endif
 
 	/*
 	 * free areas of different sizes

@@ -220,6 +224,8 @@ struct zone {
  */
 #define DEF_PRIORITY 12
 
+#define TOTAL_ZONES (MAX_NUMNODES * MAX_NR_ZONES)
+
 /*
  * One allocation request operates on a zonelist. A zonelist
  * is a list of zones, the first one is the 'goal' of the

@@ -232,10 +238,9 @@ struct zone {
  * footprint of this construct is very small.
  */
 struct zonelist {
-	struct zone *zones[MAX_NUMNODES * MAX_NR_ZONES + 1]; // NULL delimited
+	struct zone *zones[TOTAL_ZONES + 1]; // NULL delimited
 };
 
-
 /*
  * The pg_data_t structure is used in machines with CONFIG_DISCONTIGMEM
  * (mostly NUMA machines?) to denote a higher-level memory zone than the

@@ -275,6 +280,7 @@ void __get_zone_counts(unsigned long *ac
 void get_zone_counts(unsigned long *active, unsigned long *inactive,
 			unsigned long *free);
 void build_all_zonelists(void);
+void build_percpu_pagelists(void);
 void wakeup_kswapd(struct zone *zone, int order);
 int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
 		int alloc_type, int can_try_harder, int gfp_high);

261 lines not on this page · raw message

Index: linux-2.6/init/main.c
===================================================================
@@ -454,6 +454,7 @@ asmlinkage void __init start_kernel(void
 	 */
 	preempt_disable();
 	build_all_zonelists();
+	build_percpu_pagelists();
 	page_alloc_init();
 	printk(KERN_NOTICE "Kernel command line: %s\n", saved_command_line);
 	parse_early_param();

Index: linux-2.6/mm/mempolicy.c
===================================================================
@@ -721,7 +721,7 @@ static struct page *alloc_page_interleav
 	zl = NODE_DATA(nid)->node_zonelists + (gfp & GFP_ZONEMASK);
 	page = __alloc_pages(gfp, order, zl);
 	if (page && page_zone(page) == zl->zones[0]) {
-		zl->zones[0]->pageset[get_cpu()].interleave_hit++;
+		zl->zones[0]->stats[get_cpu()].interleave_hit++;
 		put_cpu();
 	}
 	return page;

Index: linux-2.6/mm/page_alloc.c
===================================================================
@@ -70,7 +70,7 @@ struct zone *zone_table[1 << (ZONES_SHIF
 EXPORT_SYMBOL(zone_table);
 
 struct zone_pagesets {
-	struct per_cpu_pageset p[TOTAL_ZONES];
+	struct per_cpu_pageset *p[TOTAL_ZONES];
 };
 
 #define this_zone_pagesets()	(&__get_cpu_var(zone_pagesets))

@@ -80,7 +80,7 @@ struct zone_pagesets {
 	(NODEZONE((zone)->zone_pgdat->node_id, zone_idx(zone)))
 
 #define zone_pageset(zp, zone)		\
-	(&zp->p[zone_pagesets_idx(zone)])
+	(zp->p[zone_pagesets_idx(zone)])
 
 /*
  * List of pointers to per_cpu_pagesets for each zone.

@@ -1579,7 +1579,8 @@ void __init build_percpu_pagelists(void)
 				struct per_cpu_pages *pcp;
 			
 				zp = cpu_zone_pagesets(cpu);
-				pageset = &zp->p[NODEZONE(nid, j)];
+				pageset = alloc_bootmem_node(pgdat, sizeof(*pageset));
+				zp->p[NODEZONE(nid, j)] = pageset;
 
 				pcp = &pageset->pcp[0];	/* hot */
 				pcp->count = 0;

Index: linux-2.6/include/linux/mmzone.h
===================================================================
@@ -128,6 +128,9 @@ struct zone {
 	struct per_cpu_zone_stats stats[NR_CPUS];
 #endif
 
+	/* global pageset for CPUs without private pagesets for this zone */
+	struct per_cpu_pageset	pageset;
+
 	/*
 	 * free areas of different sizes
 	 */

221 lines not on this page · raw message

Index: linux-2.6/mm/page_alloc.c
===================================================================
@@ -1626,14 +1626,24 @@ void __init build_percpu_pagelists(void)
 			init_percpu_pageset(&zone->pageset, batch);
 			for (cpu = 0; cpu < NR_CPUS; cpu++) {
 				struct zone_pagesets *zp;
-				struct per_cpu_pageset *pageset;
+				struct per_cpu_pageset *pageset = NULL;
 			
 				zp = cpu_zone_pagesets(cpu);
-				pageset = alloc_bootmem_node(pgdat, sizeof(*pageset));
-				init_percpu_pageset(pageset, batch);
+				
+				/*
+				 * XXX: this test could be something like
+				 *   if (node_distance <= blah)
+				 * which would allow pagesets on close
+				 * remote nodes as well as the local node.
+				 */
+				if (cpu_to_node(cpu) == nid) {
+					pageset = alloc_bootmem_node(pgdat,
+							sizeof(*pageset));
+					init_percpu_pageset(pageset, batch);
+				}
 				zp->p[NODEZONE(nid, j)] = pageset;
-
 			}
+
 			printk(KERN_DEBUG "  %s zone: %lu pages, LIFO batch:%lu\n",
 					zone_names[j], zone->present_pages, batch);
 		}

Index: linux/drivers/base/node.c
===================================================================
@@ -87,7 +87,7 @@ static ssize_t node_read_numastat(struct
 	for (i = 0; i < MAX_NR_ZONES; i++) {
 		struct zone *z = &pg->node_zones[i];
 		for (cpu = 0; cpu < NR_CPUS; cpu++) {
-			struct per_cpu_pageset *ps = &z->pageset[cpu];
+			struct per_cpu_zone_stats *ps = &z->stats[cpu];
 			numa_hit += ps->numa_hit;
 			numa_miss += ps->numa_miss;
 			numa_foreign += ps->numa_foreign;