per-zone LRUs

PATCH In review Andrew Morton 1 participant 1 message Aug 11, 2002

Subsystems (8)

Memory management MM core Page allocator Proc filesystem +4 more subsystems

lore

Share & tools

lore

Andrew Morton

author

2002-08-11 07:31 UTC #0 lore raw

PATCH 17/21 per-zone LRUs

Replace the global page LRUs with per-zone LRUs.

This fixes the failure described at
http://mail.nl.linux.org/linux-mm/2002-08/msg00049.html

It will also fix the problem wherein a search for a reclaimable ZONE_NORMAL page will undesirably move aged ZONE_HIGHMEM pages to the head of the inactive list. (I haven't tried to measure any benefit from this aspect).

It will also reduces the amount of CPU spent scanning pages in page reclaim. I haven't instrumented this either.

This is a minimal conversion - the aging and reclaim logic is left unchanged, as far as is possible.

I was bitten by the "incremental min" logic in __alloc_pages again. There's a state in which the sum-of-mins exceeds zone->pages_high. So we call into try_to_free_pages(), which does nothing at all (all zones have free_pages > pages_high). The incremental min is unchanged and the VM locks up.

This was fixed in __alloc_pages: if zone->free_pages is greater than zone->pages_high then just go and grab a page.

 fs/proc/proc_misc.c        |    9 +++-
 include/linux/mm_inline.h  |   40 ++++++++++++++++++
 include/linux/mmzone.h     |   11 ++++-
 include/linux/page-flags.h |    2 
 include/linux/swap.h       |   45 --------------------
 mm/filemap.c               |    7 +--
 mm/page_alloc.c            |   54 +++++++++++++++++-------
 mm/swap.c                  |   26 ++++-------
 mm/vmscan.c                |   98 ++++++++++++++++++++++++++-------------------
 9 files changed, 166 insertions(+), 126 deletions(-)

include/linux/mm_inline.h new file +40 -0

@@ -0,0 +1,40 @@
+
+static inline void
+add_page_to_active_list(struct zone *zone, struct page *page)
+{
+	list_add(&page->lru, &zone->active_list);
+	zone->nr_active++;
+}
+
+static inline void
+add_page_to_inactive_list(struct zone *zone, struct page *page)
+{
+	list_add(&page->lru, &zone->inactive_list);
+	zone->nr_inactive++;
+}
+
+static inline void
+del_page_from_active_list(struct zone *zone, struct page *page)
+{
+	list_del(&page->lru);
+	zone->nr_active--;
+}
+
+static inline void
+del_page_from_inactive_list(struct zone *zone, struct page *page)
+{
+	list_del(&page->lru);
+	zone->nr_inactive--;
+}
+
+static inline void
+del_page_from_lru(struct zone *zone, struct page *page)
+{
+	list_del(&page->lru);
+	if (PageActive(page)) {
+		ClearPageActive(page);
+		zone->nr_active--;
+	} else {
+		zone->nr_inactive--;
+	}
+}

include/linux/mmzone.h renamed from include/linux/mmzone.h~per-zone-lru +9 -2

@@ -8,6 +8,7 @@
 #include <linux/spinlock.h>
 #include <linux/list.h>
 #include <linux/wait.h>
+#include <asm/atomic.h>
 
 /*
  * Free memory management - zoned buddy allocator.

@@ -43,6 +44,12 @@ struct zone {
 	unsigned long		pages_min, pages_low, pages_high;
 	int			need_balance;
 
+	struct list_head	active_list;
+	struct list_head	inactive_list;
+	atomic_t		refill_counter;
+	unsigned long		nr_active;
+	unsigned long		nr_inactive;
+
 	/*
 	 * free areas of different sizes
 	 */

@@ -157,10 +164,10 @@ memclass(struct zone *pgzone, struct zon
  * prototypes for the discontig memory code.
  */
 struct page;
-extern void show_free_areas_core(pg_data_t *pgdat);
-extern void free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap,
+void free_area_init_core(int nid, pg_data_t *pgdat, struct page **gmap,
   unsigned long *zones_size, unsigned long paddr, unsigned long *zholes_size,
   struct page *pmap);
+void get_zone_counts(unsigned long *active, unsigned long *inactive);
 
 extern pg_data_t contig_page_data;

include/linux/swap.h renamed from include/linux/swap.h~per-zone-lru +0 ~~-45~~

51 lines not on this page · raw message

mm/page_alloc.c renamed from mm/page_alloc.c~per-zone-lru +38 ~~-16~~

122 lines not on this page · raw message

mm/swap.c renamed from mm/swap.c~per-zone-lru +10 ~~-16~~

@@ -19,6 +19,7 @@
 #include <linux/pagemap.h>
 #include <linux/pagevec.h>
 #include <linux/init.h>
+#include <linux/mm_inline.h>
 #include <linux/prefetch.h>
 
 /* How many pages do we try to swap or page in/out together? */

@@ -31,6 +32,7 @@ static inline void activate_page_nolock(
 {
 	if (PageLRU(page) && !PageActive(page)) {
 		del_page_from_inactive_list(page);
+		SetPageActive(page);
 		add_page_to_active_list(page);
 		KERNEL_STAT_INC(pgactivate);
 	}

@@ -83,10 +85,7 @@ void __page_cache_release(struct page *p
 		spin_lock_irqsave(&_pagemap_lru_lock, flags);
 		if (!TestClearPageLRU(page))
 			BUG();
-		if (PageActive(page))
-			del_page_from_active_list(page);
-		else
-			del_page_from_inactive_list(page);
+		del_page_from_lru(page);
 		spin_unlock_irqrestore(&_pagemap_lru_lock, flags);
 	}
 	__free_page(page);

@@ -124,12 +123,8 @@ void __pagevec_release(struct pagevec *p
 			lock_held = 1;
 		}
 
-		if (TestClearPageLRU(page)) {
-			if (PageActive(page))
-				del_page_from_active_list(page);
-			else
-				del_page_from_inactive_list(page);
-		}
+		if (TestClearPageLRU(page))
+			del_page_from_lru(page);
 		if (page_count(page) == 0)
 			pagevec_add(&pages_to_free, page);
 	}

@@ -182,8 +177,10 @@ void pagevec_deactivate_inactive(struct 
 			spin_lock_irq(&_pagemap_lru_lock);
 			lock_held = 1;
 		}
-		if (!PageActive(page) && PageLRU(page))
-			list_move(&page->lru, &inactive_list);
+		if (!PageActive(page) && PageLRU(page)) {
+			struct zone *zone = page_zone(page);
+			list_move(&page->lru, &zone->inactive_list);
+		}
 	}
 	if (lock_held)
 		spin_unlock_irq(&_pagemap_lru_lock);

@@ -224,10 +221,7 @@ void __pagevec_lru_del(struct pagevec *p
 
 		if (!TestClearPageLRU(page))
 			BUG();
-		if (PageActive(page))
-			del_page_from_active_list(page);
-		else
-			del_page_from_inactive_list(page);
+		del_page_from_lru(page);
 	}
 	spin_unlock_irq(&_pagemap_lru_lock);
 	pagevec_release(pvec);

mm/vmscan.c renamed from mm/vmscan.c~per-zone-lru +58 ~~-40~~

219 lines not on this page · raw message

include/linux/page-flags.h renamed from include/linux/page-flags.h~per-zone-lru +0 -2

@@ -74,8 +74,6 @@ extern struct page_state {
 	unsigned long nr_dirty;
 	unsigned long nr_writeback;
 	unsigned long nr_pagecache;
-	unsigned long nr_active;	/* on active_list LRU */
-	unsigned long nr_inactive;	/* on inactive_list LRU */
 	unsigned long nr_page_table_pages;
 	unsigned long nr_reverse_maps;
 } ____cacheline_aligned_in_smp page_states[NR_CPUS];

fs/proc/proc_misc.c renamed from fs/proc/proc_misc.c~per-zone-lru +7 -2

@@ -27,6 +27,7 @@
 #include <linux/ioport.h>
 #include <linux/config.h>
 #include <linux/mm.h>
+#include <linux/mmzone.h>
 #include <linux/pagemap.h>
 #include <linux/swap.h>
 #include <linux/slab.h>

@@ -134,8 +135,12 @@ static int meminfo_read_proc(char *page,
 	struct sysinfo i;
 	int len, committed;
 	struct page_state ps;
+	unsigned long inactive;
+	unsigned long active;
 
 	get_page_state(&ps);
+	get_zone_counts(&active, &inactive);
+
 /*
  * display in kilobytes.
  */

@@ -171,8 +176,8 @@ static int meminfo_read_proc(char *page,
 		K(i.sharedram),
 		K(ps.nr_pagecache-swapper_space.nrpages),
 		K(swapper_space.nrpages),
-		K(ps.nr_active),
-		K(ps.nr_inactive),
+		K(active),
+		K(inactive),
 		K(i.totalhigh),
 		K(i.freehigh),
 		K(i.totalram-i.totalhigh),

mm/filemap.c renamed from mm/filemap.c~per-zone-lru +4 -3

@@ -1158,14 +1158,15 @@ do_readahead(struct file *file, unsigned
 {
 	struct address_space *mapping = file->f_dentry->d_inode->i_mapping;
 	unsigned long max;
-	struct page_state ps;
+	unsigned long active;
+	unsigned long inactive;
 
 	if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
 		return -EINVAL;
 
 	/* Limit it to a sane percentage of the inactive list.. */
-	get_page_state(&ps);
-	max = ps.nr_inactive / 2;
+	get_zone_counts(&active, &inactive);
+	max = inactive / 2;
 	if (nr > max)
 		nr = max;