Skip to contentSkip to conversation

lazy_buddy-2.5.25-1

PATCH William Lee Irwin III 1 participant 6 messages Jul 10, 2002 – Jul 13, 2002

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

This patch implements deferred coalescing for the Linux page-level allocator, or the lazy buddy algorithm. This appears to reduce the allocator overhead in some AF_UNIX and local TCP connection scenarios according to LMbench (hopefully I can follow up with numbers soon).

This is a simple forward port of the 2.5.23-based version decoupled from some of the cleanups previously bundled with it.

TODO:

(1)  make the buddy and deferred lists singly-linked
(2)  convert /proc/fraginfo to seq_file()
(3)  find a better home for /proc/fraginfo somewhere in /proc/
(4)  analyze for cache performance
(5)  use split_free_pages() to recover pages from higher-order
	deferred queues to avoid spuriously failing under fragmentation
(6)  get real benchmark numbers out of the thing
(7)  collect statistics on allocation rates for various orders
(8)  collect statistics on allocation failures for various orders
(9)  collect statistics on allocation failures due to fragmentation

(10) collect qualitative information on long-term fragmentation

Follow-on patch TODO item:
(11) gang preassembly and bulk transfer to and from per-cpu pools

This "release" doesn't feature any real development; it's mostly just a forward port. In fact, it removes various cleanups tied to prior releases. For some, this may provide some clarification as to what the additional mechanics involved are. Gang allocation support will be decoupled from this, and will be my main focus in the allocator before coming back to this again.

This patch may also be found at:

ftp://ftp.kernel.org/pub/linux/kernel/people/wli/vm/lazy_buddy/lazy_buddy-2.5.25-1 bk://linux-wli.bkbits.net/lazy_buddy/

Cheers,
Bill

fs/proc/proc_misc.c +22 -2
diff -Nur --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5/fs/proc/proc_misc.c lazy_buddy/fs/proc/proc_misc.c
@@ -132,6 +132,7 @@
 	struct sysinfo i;
 	int len;
 	struct page_state ps;
+	extern unsigned long nr_deferred_pages(void);
 
 	get_page_state(&ps);
 /*
@@ -159,7 +160,8 @@
 		"SwapTotal:    %8lu kB\n"
 		"SwapFree:     %8lu kB\n"
 		"Dirty:        %8lu kB\n"
-		"Writeback:    %8lu kB\n",
+		"Writeback:    %8lu kB\n"
+		"Deferred:     %8lu kB\n",
 		K(i.totalram),
 		K(i.freeram),
 		K(i.sharedram),
@@ -174,13 +176,30 @@
 		K(i.totalswap),
 		K(i.freeswap),
 		K(ps.nr_dirty),
-		K(ps.nr_writeback)
+		K(ps.nr_writeback),
+		K(nr_deferred_pages())
 		);
 
 	return proc_calc_metrics(page, start, off, count, eof, len);
 #undef K
 }
 
+extern int fragmentation_stats(char *, int);
+static int fragmentation_read_proc(	char *page,
+					char **start,
+					off_t off,
+					int count,
+					int *eof,
+					void *data)
+{
+	int nid, len = 0;
+
+	for (nid = 0; nid < numnodes; ++nid)
+		len += fragmentation_stats(page + len, nid);
+
+	return proc_calc_metrics(page, start, off, count, eof, len);
+}
+
 static int version_read_proc(char *page, char **start, off_t off,
 				 int count, int *eof, void *data)
 {
@@ -550,6 +569,7 @@
 		{"loadavg",     loadavg_read_proc},
 		{"uptime",	uptime_read_proc},
 		{"meminfo",	meminfo_read_proc},
+		{"fraginfo",	fragmentation_read_proc},
 		{"version",	version_read_proc},
 #ifdef CONFIG_PROC_HARDWARE
 		{"hardware",	hardware_read_proc},
include/linux/mmzone.h +3 -2
diff -Nur --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5/include/linux/mmzone.h lazy_buddy/include/linux/mmzone.h
@@ -20,8 +20,9 @@
 #endif
 
 typedef struct free_area_struct {
-	struct list_head	free_list;
-	unsigned long		*map;
+	unsigned long	globally_free, active, locally_free;
+	list_t		free_list, deferred_pages;
+	unsigned long	*map;
 } free_area_t;
 
 struct pglist_data;
mm/bootmem.c +4 -0
diff -Nur --exclude=SCCS --exclude=BitKeeper --exclude=ChangeSet linux-2.5/mm/bootmem.c lazy_buddy/mm/bootmem.c
@@ -242,6 +242,7 @@
 	return ret;
 }
 
+extern void forced_coalesce(zone_t *zone);
 static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
 {
 	struct page *page = pgdat->node_mem_map;
@@ -288,6 +289,9 @@
 	}
 	total += count;
 	bdata->node_bootmem_map = NULL;
+
+	for (i = 0; i < MAX_NR_ZONES; ++i)
+		forced_coalesce(&pgdat->node_zones[i]);
 
 	return total;
 }
mm/page_alloc.c +233 -22

383 lines not on this page · raw message

#1 lore raw
Quoted 6 lines · William Lee Irwin III

This patch implements deferred coalescing for the Linux page-level allocator, or the lazy buddy algorithm. This appears to reduce the allocator overhead in some AF_UNIX and local TCP connection scenarios according to LMbench (hopefully I can follow up with numbers soon). This is a simple forward port of the 2.5.23-based version decoupled from some of the cleanups previously bundled with it.

Ugh, forward port breakage. Ran into this in a different patch but realized it was broken here too. Sorry folks. You'll need this on top of the prior post.

Cheers,
Bill

===== mm/page_alloc.c 1.125 vs edited =====

mm/page_alloc.c +1 -2
@@ -312,7 +312,6 @@
 
 			if (bad_range(zone, page))
 				BUG();
-			prep_new_page(page);
 			return page;	
 		}
 		curr_order++;
@@ -394,7 +393,7 @@
 		page = buddy_alloc(zone, order);
 	if (unlikely(!page))
 		goto out;
-	set_page_count(page, 1);
+	prep_new_page(page);
 	area->active++;
 	zone->free_pages -= 1UL << order;
 out:
#2 lore raw
Quoted 3 lines · William Lee Irwin III

Ugh, forward port breakage. Ran into this in a different patch but realized it was broken here too. Sorry folks. You'll need this on top of the prior post.

Even worse, I spotted another (thankfully more minor) bug while still peeking at this... okay, back to more urgent things.

Cheers,
Bill

# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.631 -> 1.632
# mm/page_alloc.c 1.126 -> 1.127
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 02/07/10 [email protected] 1.632
# page_alloc.c:
# Correct nr_deferred_pages() calculation.
# --------------------------------------------
#

mm/page_alloc.c +2 -2
diff --minimal -Nru a/mm/page_alloc.c b/mm/page_alloc.c
@@ -739,8 +739,8 @@
 	for (pgdat = pgdat_list; pgdat; pgdat = pgdat->node_next) {
 		node_zones = pgdat->node_zones;
 		for (i = 0; i < MAX_NR_ZONES; ++i) {
-			for (order = 0; order < MAX_ORDER; ++order)
-				pages += node_zones[i].free_area[order].locally_free;
+			for (order = MAX_ORDER; order >= 0; --order)
+				pages = 2*pages + node_zones[i].free_area[order].locally_free;
 		}
 	}
 	return pages;
#3 lore raw
Quoted 2 lines · William Lee Irwin III

Even worse, I spotted another (thankfully more minor) bug while still peeking at this... okay, back to more urgent things.

I'm *not* having a good day. One parting shot and I really go back to the other stuff:

Review line 746

Quoted 11 lines
(quoted diff) +2 -2
@@ -739,8 +739,8 @@
 	for (pgdat = pgdat_list; pgdat; pgdat = pgdat->node_next) {
 		node_zones = pgdat->node_zones;
 		for (i = 0; i < MAX_NR_ZONES; ++i) {
-			for (order = 0; order < MAX_ORDER; ++order)
-				pages += node_zones[i].free_area[order].locally_free;
+			for (order = MAX_ORDER; order >= 0; --order)
+				pages = 2*pages + node_zones[i].free_area[order].locally_free;
 		}
 	}
 	return pages;

forgets to start from 0 pages for each zone. Don't bother trying to be smart and let the compiler figure it out, it's for /proc/meminfo anyway:

Cheers,
Bill

# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:
# ChangeSet 1.633 -> 1.634
# mm/page_alloc.c 1.128 -> 1.129
#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 02/07/10 [email protected] 1.634
# page_alloc.c:
# Correct nr_deferred_pages() calculation.
# --------------------------------------------
#

mm/page_alloc.c +1 -1
diff --minimal -Nru a/mm/page_alloc.c b/mm/page_alloc.c
@@ -740,7 +740,7 @@
 		node_zones = pgdat->node_zones;
 		for (i = 0; i < MAX_NR_ZONES; ++i) {
 			for (order = 0; order < MAX_ORDER; ++order)
-				pages += node_zones[i].free_area[order].locally_free;
+				pages += node_zones[i].free_area[order].locally_free << order;
 		}
 	}
 	return pages;
#4 lore raw
In reply to William Lee Irwin III
Quoted 1 line · William Lee Irwin III

This patch implements deferred coalescing for the Linux page-level

[...]

Quoted 1 line

TODO:

[...]

Quoted 2 lines

(5) use split_free_pages() to recover pages from higher-order deferred queues to avoid spuriously failing under fragmentation

Okay, people bugged me about this so here it is, testbooted on an 8cpu 16GB i386 box:

Cheers,
Bill

===== mm/page_alloc.c 1.129 vs edited =====

mm/page_alloc.c +25 -1
@@ -344,6 +344,26 @@
 	}
 }
 
+static struct page *FASTCALL(steal_deferred_page(zone_t *, int));
+static struct page *steal_deferred_page(zone_t *zone, int order)
+{
+	struct page *page;
+	free_area_t *area = zone->free_area;
+	int found_order;
+
+	for (found_order = order+1; found_order < MAX_ORDER; ++found_order)
+		if (!list_empty(&area[found_order].deferred_pages))
+			goto found_page;
+
+	return NULL;
+
+found_page:
+	page = list_entry(area[found_order].deferred_pages.next, struct page, list);
+	list_del(&page->list);
+	area[found_order].locally_free--;
+	split_pages(zone, page, order, found_order);
+}
+
 #ifdef CONFIG_SOFTWARE_SUSPEND
 int is_head_of_free_region(struct page *page)
 {
@@ -391,8 +411,12 @@
 		area->locally_free--;
 	} else
 		page = buddy_alloc(zone, order);
-	if (unlikely(!page))
+	if (unlikely(!page)) {
+		page = steal_deferred_page(zone, order);
+		if (likely(!page))
 		goto out;
+	}
+
 	prep_new_page(page);
 	area->active++;
 	zone->free_pages -= 1UL << order;
#5 lore raw
Quoted 2 lines · William Lee Irwin III

Okay, people bugged me about this so here it is, testbooted on an 8cpu 16GB i386 box:

A little birdie told me I forgot the following on top of this:

Cheers,
Bill

===== mm/page_alloc.c 1.130 vs edited =====

mm/page_alloc.c +1 -0
@@ -362,6 +362,7 @@
 	list_del(&page->list);
 	area[found_order].locally_free--;
 	split_pages(zone, page, order, found_order);
+	return page;
 }
 
 #ifdef CONFIG_SOFTWARE_SUSPEND