mm: 2.6.17-pr1 - generic page-replacement framework and 4 new policies

Index: linux-2.6/include/linux/swap.h
===================================================================
@@ -165,7 +165,6 @@ extern unsigned int nr_free_pagecache_pa
 /* linux/mm/swap.c */
 extern void FASTCALL(lru_cache_add(struct page *));
 extern void FASTCALL(lru_cache_add_active(struct page *));
-extern void FASTCALL(activate_page(struct page *));
 extern void FASTCALL(mark_page_accessed(struct page *));
 extern void lru_add_drain(void);
 extern int lru_add_drain_all(void);

Index: linux-2.6/mm/swapfile.c
===================================================================
@@ -496,7 +496,7 @@ static void unuse_pte(struct vm_area_str
 	 * Move the page to the active list so it is not
 	 * immediately swapped out again after swapon.
 	 */
-	activate_page(page);
+	mark_page_accessed(page);
 }
 
 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,

@@ -598,7 +598,7 @@ static int unuse_mm(struct mm_struct *mm
 		 * Activate page so shrink_cache is unlikely to unmap its
 		 * ptes while lock is dropped, so swapoff can make progress.
 		 */
-		activate_page(page);
+		mark_page_accessed(page);
 		unlock_page(page);
 		down_read(&mm->mmap_sem);
 		lock_page(page);

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -61,6 +61,8 @@ struct scan_control {
 	 * In this context, it doesn't matter that we scan the
 	 * whole list at once. */
 	int swap_cluster_max;
+
+	unsigned long nr_writeout;	/* page against which writeout was started */
 };
 
 /*

@@ -407,6 +409,7 @@ static unsigned long shrink_page_list(st
 	struct pagevec freed_pvec;
 	int pgactivate = 0;
 	unsigned long nr_reclaimed = 0;
+	int writeout = 0;
 
 	cond_resched();
 

@@ -488,8 +491,10 @@ static unsigned long shrink_page_list(st
 			case PAGE_ACTIVATE:
 				goto activate_locked;
 			case PAGE_SUCCESS:
-				if (PageWriteback(page) || PageDirty(page))
+				if (PageWriteback(page) || PageDirty(page)) {
+					writeout++;
 					goto keep;
+				}
 				/*
 				 * A synchronous write - probably a ramdisk.  Go
 				 * ahead and try to reclaim the page.

@@ -555,6 +560,7 @@ keep:
 	if (pagevec_count(&freed_pvec))
 		__pagevec_release_nonlru(&freed_pvec);
 	mod_page_state(pgactivate, pgactivate);
+	sc->nr_writeout += writeout;
 	return nr_reclaimed;
 }
 

@@ -1123,6 +1129,7 @@ scan:
 		 * pages behind kswapd's direction of progress, which would
 		 * cause too much scanning of the lower zones.
 		 */
+		sc.nr_writeout = 0;
 		for (i = 0; i <= end_zone; i++) {
 			struct zone *zone = pgdat->node_zones + i;
 			int nr_slab;

@@ -1170,7 +1177,7 @@ scan:
 		 * OK, kswapd is getting into trouble.  Take a nap, then take
 		 * another pass across the zones.
 		 */
-		if (total_scanned && priority < DEF_PRIORITY - 2)
+		if (sc.nr_writeout > SWAP_CLUSTER_MAX/2)
 			blk_congestion_wait(WRITE, HZ/10);
 
 		/*

Index: linux-2.6/mm/useonce.c
===================================================================
@@ -0,0 +1,3 @@
+
+
+

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -0,0 +1,21 @@
+#ifndef _LINUX_MM_USEONCE_POLICY_H
+#define _LINUX_MM_USEONCE_POLICY_H
+
+#ifdef __KERNEL__
+
+static inline void pgrep_hint_active(struct page *page)
+{
+	SetPageActive(page);
+}
+
+static inline void
+__pgrep_add(struct zone *zone, struct page *page)
+{
+	if (PageActive(page))
+		add_page_to_active_list(zone, page);
+	else
+		add_page_to_inactive_list(zone, page);
+}
+
+#endif /* __KERNEL__ */
+#endif /* _LINUX_MM_USEONCE_POLICY_H */

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -0,0 +1,38 @@
+#ifndef _LINUX_MM_PAGE_REPLACE_H
+#define _LINUX_MM_PAGE_REPLACE_H
+
+#ifdef __KERNEL__
+
+#include <linux/mmzone.h>
+#include <linux/mm.h>
+#include <linux/pagevec.h>
+#include <linux/mm_inline.h>
+
+/* void pgrep_hint_active(struct page *); */
+extern void fastcall pgrep_add(struct page *);
+/* void __pgrep_add(struct zone *, struct page *); */
+/* void pgrep_add_drain(void); */
+extern void __pgrep_add_drain(unsigned int);
+extern int pgrep_add_drain_all(void);
+extern void __pagevec_pgrep_add(struct pagevec *);
+
+#ifdef CONFIG_MM_POLICY_USEONCE
+#include <linux/mm_use_once_policy.h>
+#else
+#error no mm policy
+#endif
+
+static inline void pagevec_pgrep_add(struct pagevec *pvec)
+{
+	if (pagevec_count(pvec))
+		__pagevec_pgrep_add(pvec);
+}
+
+static inline void pgrep_add_drain(void)
+{
+	__pgrep_add_drain(get_cpu());
+	put_cpu();
+}
+
+#endif /* __KERNEL__ */
+#endif /* _LINUX_MM_PAGE_REPLACE_H */

Index: linux-2.6/mm/filemap.c
===================================================================
@@ -30,6 +30,7 @@
 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/cpuset.h>
+#include <linux/mm_page_replace.h>
 #include "filemap.h"
 #include "internal.h"
 

@@ -430,7 +431,7 @@ int add_to_page_cache_lru(struct page *p
 {
 	int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
 	if (ret == 0)
-		lru_cache_add(page);
+		pgrep_add(page);
 	return ret;
 }
 

@@ -1784,7 +1785,7 @@ repeat:
 			page = *cached_page;
 			page_cache_get(page);
 			if (!pagevec_add(lru_pvec, page))
-				__pagevec_lru_add(lru_pvec);
+				__pagevec_pgrep_add(lru_pvec);
 			*cached_page = NULL;
 		}
 	}

@@ -2114,7 +2115,7 @@ generic_file_buffered_write(struct kiocb
 	if (unlikely(file->f_flags & O_DIRECT) && written)
 		status = filemap_write_and_wait(mapping);
 
-	pagevec_lru_add(&lru_pvec);
+	pagevec_pgrep_add(&lru_pvec);
 	return written ? written : status;
 }
 EXPORT_SYMBOL(generic_file_buffered_write);

111 lines not on this page · raw message

Index: linux-2.6/mm/memory.c
===================================================================
@@ -48,6 +48,7 @@
 #include <linux/rmap.h>
 #include <linux/module.h>
 #include <linux/init.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/pgalloc.h>
 #include <asm/uaccess.h>

@@ -870,7 +871,7 @@ unsigned long zap_page_range(struct vm_a
 	unsigned long end = address + size;
 	unsigned long nr_accounted = 0;
 
-	lru_add_drain();
+	pgrep_add_drain();
 	tlb = tlb_gather_mmu(mm, 0);
 	update_hiwater_rss(mm);
 	end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);

@@ -1505,7 +1506,8 @@ gotten:
 		ptep_establish(vma, address, page_table, entry);
 		update_mmu_cache(vma, address, entry);
 		lazy_mmu_prot_update(entry);
-		lru_cache_add_active(new_page);
+		pgrep_hint_active(new_page);
+		pgrep_add(new_page);
 		page_add_new_anon_rmap(new_page, vma, address);
 
 		/* Free the old page.. */

@@ -1857,7 +1859,7 @@ void swapin_readahead(swp_entry_t entry,
 		}
 #endif
 	}
-	lru_add_drain();	/* Push any new pages onto the LRU now */
+	pgrep_add_drain();	/* Push any new pages onto the LRU now */
 }
 
 /*

@@ -1991,7 +1993,8 @@ static int do_anonymous_page(struct mm_s
 		if (!pte_none(*page_table))
 			goto release;
 		inc_mm_counter(mm, anon_rss);
-		lru_cache_add_active(page);
+		pgrep_hint_active(page);
+		pgrep_add(page);
 		page_add_new_anon_rmap(page, vma, address);
 	} else {
 		/* Map the ZERO_PAGE - vm_page_prot is readonly */

@@ -2122,7 +2125,8 @@ retry:
 		set_pte_at(mm, address, page_table, entry);
 		if (anon) {
 			inc_mm_counter(mm, anon_rss);
-			lru_cache_add_active(new_page);
+			pgrep_hint_active(new_page);
+			pgrep_add(new_page);
 			page_add_new_anon_rmap(new_page, vma, address);
 		} else {
 			inc_mm_counter(mm, file_rss);

Index: linux-2.6/mm/mmap.c
===================================================================
@@ -25,6 +25,7 @@
 #include <linux/mount.h>
 #include <linux/mempolicy.h>
 #include <linux/rmap.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>

@@ -1662,7 +1663,7 @@ static void unmap_region(struct mm_struc
 	struct mmu_gather *tlb;
 	unsigned long nr_accounted = 0;
 
-	lru_add_drain();
+	pgrep_add_drain();
 	tlb = tlb_gather_mmu(mm, 0);
 	update_hiwater_rss(mm);
 	unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);

@@ -1942,7 +1943,7 @@ void exit_mmap(struct mm_struct *mm)
 	unsigned long nr_accounted = 0;
 	unsigned long end;
 
-	lru_add_drain();
+	pgrep_add_drain();
 	flush_cache_mm(mm);
 	tlb = tlb_gather_mmu(mm, 1);
 	/* Don't update_hiwater_rss(mm) here, do_exit already did */

Index: linux-2.6/mm/shmem.c
===================================================================
@@ -954,7 +954,7 @@ struct page *shmem_swapin(struct shmem_i
 			break;
 		page_cache_release(page);
 	}
-	lru_add_drain();	/* Push any new pages onto the LRU now */
+	pgrep_add_drain();	/* Push any new pages onto the LRU now */
 	return shmem_swapin_async(p, entry, idx);
 }
 

165 lines not on this page · raw message

Index: linux-2.6/mm/swap_state.c
===================================================================
@@ -16,6 +16,7 @@
 #include <linux/backing-dev.h>
 #include <linux/pagevec.h>
 #include <linux/migrate.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/pgtable.h>
 

@@ -276,7 +277,7 @@ void free_pages_and_swap_cache(struct pa
 {
 	struct page **pagep = pages;
 
-	lru_add_drain();
+	pgrep_add_drain();
 	while (nr) {
 		int todo = min(nr, PAGEVEC_SIZE);
 		int i;

@@ -354,7 +355,8 @@ struct page *read_swap_cache_async(swp_e
 			/*
 			 * Initiate read into locked page and return.
 			 */
-			lru_cache_add_active(new_page);
+			pgrep_hint_active(new_page);
+			pgrep_add(new_page);
 			swap_readpage(NULL, new_page);
 			return new_page;
 		}

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -34,6 +34,7 @@
 #include <linux/notifier.h>
 #include <linux/rwsem.h>
 #include <linux/delay.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>

@@ -630,7 +631,7 @@ static unsigned long shrink_inactive_lis
 
 	pagevec_init(&pvec, 1);
 
-	lru_add_drain();
+	pgrep_add_drain();
 	spin_lock_irq(&zone->lru_lock);
 	do {
 		struct page *page;

@@ -757,7 +758,7 @@ static void shrink_active_list(unsigned 
 			reclaim_mapped = 1;
 	}
 
-	lru_add_drain();
+	pgrep_add_drain();
 	spin_lock_irq(&zone->lru_lock);
 	pgmoved = isolate_lru_pages(nr_pages, &zone->active_list,
 				    &l_hold, &pgscanned);

Index: linux-2.6/fs/cifs/file.c
===================================================================
@@ -30,6 +30,7 @@
 #include <linux/smp_lock.h>
 #include <linux/writeback.h>
 #include <linux/delay.h>
+#include <linux/mm_page_replace.h>
 #include <asm/div64.h>
 #include "cifsfs.h"
 #include "cifspdu.h"

@@ -1654,7 +1655,7 @@ static void cifs_copy_cache_pages(struct
 		SetPageUptodate(page);
 		unlock_page(page);
 		if (!pagevec_add(plru_pvec, page))
-			__pagevec_lru_add(plru_pvec);
+			__pagevec_pgrep_add(plru_pvec);
 		data += PAGE_CACHE_SIZE;
 	}
 	return;

@@ -1808,7 +1809,7 @@ static int cifs_readpages(struct file *f
 		bytes_read = 0;
 	}
 
-	pagevec_lru_add(&lru_pvec);
+	pagevec_pgrep_add(&lru_pvec);
 
 /* need to free smb_read_data buf before exit */
 	if (smb_read_data) {

Index: linux-2.6/fs/mpage.c
===================================================================
@@ -26,6 +26,7 @@
 #include <linux/writeback.h>
 #include <linux/backing-dev.h>
 #include <linux/pagevec.h>
+#include <linux/mm_page_replace.h>
 
 /*
  * I/O completion handler for multipage BIOs.

@@ -408,12 +409,12 @@ mpage_readpages(struct address_space *ma
 					&first_logical_block,
 					get_block);
 			if (!pagevec_add(&lru_pvec, page))
-				__pagevec_lru_add(&lru_pvec);
+				__pagevec_pgrep_add(&lru_pvec);
 		} else {
 			page_cache_release(page);
 		}
 	}
-	pagevec_lru_add(&lru_pvec);
+	pagevec_pgrep_add(&lru_pvec);
 	BUG_ON(!list_empty(pages));
 	if (bio)
 		mpage_bio_submit(READ, bio);

Index: linux-2.6/fs/ntfs/file.c
===================================================================
@@ -441,7 +441,7 @@ static inline int __ntfs_grab_cache_page
 			pages[nr] = *cached_page;
 			page_cache_get(*cached_page);
 			if (unlikely(!pagevec_add(lru_pvec, *cached_page)))
-				__pagevec_lru_add(lru_pvec);
+				__pagevec_pgrep_add(lru_pvec);
 			*cached_page = NULL;
 		}
 		index++;

@@ -2111,7 +2111,7 @@ err_out:
 						OSYNC_METADATA|OSYNC_DATA);
 		}
   	}
-	pagevec_lru_add(&lru_pvec);
+	pagevec_pgrep_add(&lru_pvec);
 	ntfs_debug("Done.  Returning %s (written 0x%lx, status %li).",
 			written ? "written" : "status", (unsigned long)written,
 			(long)status);

Index: linux-2.6/mm/readahead.c
===================================================================
@@ -14,6 +14,7 @@
 #include <linux/blkdev.h>
 #include <linux/backing-dev.h>
 #include <linux/pagevec.h>
+#include <linux/mm_page_replace.h>
 
 void default_unplug_io_fn(struct backing_dev_info *bdi, struct page *page)
 {

@@ -146,7 +147,7 @@ int read_cache_pages(struct address_spac
 		}
 		ret = filler(data, page);
 		if (!pagevec_add(&lru_pvec, page))
-			__pagevec_lru_add(&lru_pvec);
+			__pagevec_pgrep_add(&lru_pvec);
 		if (ret) {
 			while (!list_empty(pages)) {
 				struct page *victim;

@@ -158,7 +159,7 @@ int read_cache_pages(struct address_spac
 			break;
 		}
 	}
-	pagevec_lru_add(&lru_pvec);
+	pagevec_pgrep_add(&lru_pvec);
 	return ret;
 }
 

@@ -185,13 +186,13 @@ static int read_pages(struct address_spa
 			ret = mapping->a_ops->readpage(filp, page);
 			if (ret != AOP_TRUNCATED_PAGE) {
 				if (!pagevec_add(&lru_pvec, page))
-					__pagevec_lru_add(&lru_pvec);
+					__pagevec_pgrep_add(&lru_pvec);
 				continue;
 			} /* else fall through to release */
 		}
 		page_cache_release(page);
 	}
-	pagevec_lru_add(&lru_pvec);
+	pagevec_pgrep_add(&lru_pvec);
 	ret = 0;
 out:
 	return ret;

Index: linux-2.6/fs/exec.c
===================================================================
@@ -49,6 +49,7 @@
 #include <linux/rmap.h>
 #include <linux/acct.h>
 #include <linux/cn_proc.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/uaccess.h>
 #include <asm/mmu_context.h>

@@ -321,7 +322,8 @@ void install_arg_page(struct vm_area_str
 		goto out;
 	}
 	inc_mm_counter(mm, anon_rss);
-	lru_cache_add_active(page);
+	pgrep_hint_active(page);
+	pgrep_add(page);
 	set_pte_at(mm, address, pte, pte_mkdirty(pte_mkwrite(mk_pte(
 					page, vma->vm_page_prot))));
 	page_add_new_anon_rmap(page, vma, address);

Index: linux-2.6/include/linux/pagevec.h
===================================================================
@@ -23,8 +23,6 @@ struct pagevec {
 void __pagevec_release(struct pagevec *pvec);
 void __pagevec_release_nonlru(struct pagevec *pvec);
 void __pagevec_free(struct pagevec *pvec);
-void __pagevec_lru_add(struct pagevec *pvec);
-void __pagevec_lru_add_active(struct pagevec *pvec);
 void pagevec_strip(struct pagevec *pvec);
 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
 		pgoff_t start, unsigned nr_pages);

@@ -81,10 +79,4 @@ static inline void pagevec_free(struct p
 		__pagevec_free(pvec);
 }
 
-static inline void pagevec_lru_add(struct pagevec *pvec)
-{
-	if (pagevec_count(pvec))
-		__pagevec_lru_add(pvec);
-}
-
 #endif /* _LINUX_PAGEVEC_H */

Index: linux-2.6/include/linux/swap.h
===================================================================
@@ -163,13 +163,11 @@ extern unsigned int nr_free_buffer_pages
 extern unsigned int nr_free_pagecache_pages(void);
 
 /* linux/mm/swap.c */
-extern void FASTCALL(lru_cache_add(struct page *));
-extern void FASTCALL(lru_cache_add_active(struct page *));
 extern void FASTCALL(mark_page_accessed(struct page *));
-extern void lru_add_drain(void);
 extern int lru_add_drain_all(void);
 extern int rotate_reclaimable_page(struct page *page);
 extern void swap_setup(void);
+extern void release_pages(struct page **, int, int);
 
 /* linux/mm/vmscan.c */
 extern unsigned long try_to_free_pages(struct zone **, gfp_t);

Index: linux-2.6/fs/ramfs/file-nommu.c
===================================================================
@@ -108,7 +108,7 @@ static int ramfs_nommu_expand_for_mappin
 			goto add_error;
 
 		if (!pagevec_add(&lru_pvec, page))
-			__pagevec_lru_add(&lru_pvec);
+			__pagevec_pgrep_add(&lru_pvec);
 
 		unlock_page(page);
 	}

Index: linux-2.6/mm/mempolicy.c
===================================================================
@@ -87,6 +87,7 @@
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>
 #include <linux/migrate.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>

Index: linux-2.6/fs/splice.c
===================================================================
@@ -21,13 +21,13 @@
 #include <linux/file.h>
 #include <linux/pagemap.h>
 #include <linux/pipe_fs_i.h>
-#include <linux/mm_inline.h>
 #include <linux/swap.h>
 #include <linux/writeback.h>
 #include <linux/buffer_head.h>
 #include <linux/module.h>
 #include <linux/syscalls.h>
 #include <linux/uio.h>
+#include <linux/mm_page_replace.h>
 
 struct partial_page {
 	unsigned int offset;

@@ -587,7 +587,7 @@ static int pipe_to_file(struct pipe_inod
 		page_cache_get(page);
 
 		if (!(buf->flags & PIPE_BUF_FLAG_LRU))
-			lru_cache_add(page);
+			pgrep_add(page);
 	} else {
 find_page:
 		page = find_lock_page(mapping, index);

Index: linux-2.6/mm/migrate.c
===================================================================
@@ -24,6 +24,7 @@
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/swapops.h>
+#include <linux/mm_page_replace.h>
 
 #include "internal.h"
 

@@ -80,7 +81,7 @@ int migrate_prep(void)
 	 * drained them. Those pages will fail to migrate like other
 	 * pages that may be busy.
 	 */
-	lru_add_drain_all();
+	pgrep_add_drain();
 
 	return 0;
 }

@@ -88,16 +89,7 @@ int migrate_prep(void)
 static inline void move_to_lru(struct page *page)
 {
 	list_del(&page->lru);
-	if (PageActive(page)) {
-		/*
-		 * lru_cache_add_active checks that
-		 * the PG_active bit is off.
-		 */
-		ClearPageActive(page);
-		lru_cache_add_active(page);
-	} else {
-		lru_cache_add(page);
-	}
+	pgrep_add(page);
 	put_page(page);
 }
 

Index: linux-2.6/mm/filemap.c
===================================================================
@@ -412,6 +412,18 @@ int add_to_page_cache(struct page *page,
 		error = radix_tree_insert(&mapping->page_tree, offset, page);
 		if (!error) {
 			page_cache_get(page);
+			/*
+			 * shmem_getpage()
+			 *   lookup_swap_cache()
+			 *   TestSetPageLocked()
+			 *   move_from_swap_cache()
+			 *     add_to_page_cache()
+			 *
+			 * That path calls us with a LRU page instead of a new
+			 * page. Don't set the hint for LRU pages.
+			 */
+			if (!PageLocked(page))
+				pgrep_hint_use_once(page);
 			SetPageLocked(page);
 			page->mapping = mapping;
 			page->index = offset;

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -9,6 +9,7 @@
 #include <linux/mm_inline.h>
 
 /* void pgrep_hint_active(struct page *); */
+/* void pgrep_hint_use_once(struct page *); */
 extern void fastcall pgrep_add(struct page *);
 /* void __pgrep_add(struct zone *, struct page *); */
 /* void pgrep_add_drain(void); */

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -8,6 +8,10 @@ static inline void pgrep_hint_active(str
 	SetPageActive(page);
 }
 
+static inline void pgrep_hint_use_once(struct page *page)
+{
+}
+
 static inline void
 __pgrep_add(struct zone *zone, struct page *page)
 {

Index: linux-2.6/mm/useonce.c
===================================================================
@@ -11,64 +11,6 @@
 static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, };
 static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, };
 
-/*
- * Add the passed pages to the LRU, then drop the caller's refcount
- * on them.  Reinitialises the caller's pagevec.
- */
-void __pagevec_pgrep_add(struct pagevec *pvec)
-{
-	int i;
-	struct zone *zone = NULL;
-
-	for (i = 0; i < pagevec_count(pvec); i++) {
-		struct page *page = pvec->pages[i];
-		struct zone *pagezone = page_zone(page);
-
-		if (pagezone != zone) {
-			if (zone)
-				spin_unlock_irq(&zone->lru_lock);
-			zone = pagezone;
-			spin_lock_irq(&zone->lru_lock);
-		}
-		BUG_ON(PageLRU(page));
-		SetPageLRU(page);
-		add_page_to_inactive_list(zone, page);
-	}
-	if (zone)
-		spin_unlock_irq(&zone->lru_lock);
-	release_pages(pvec->pages, pvec->nr, pvec->cold);
-	pagevec_reinit(pvec);
-}
-
-EXPORT_SYMBOL(__pagevec_pgrep_add);
-
-static void __pagevec_lru_add_active(struct pagevec *pvec)
-{
-	int i;
-	struct zone *zone = NULL;
-
-	for (i = 0; i < pagevec_count(pvec); i++) {
-		struct page *page = pvec->pages[i];
-		struct zone *pagezone = page_zone(page);
-
-		if (pagezone != zone) {
-			if (zone)
-				spin_unlock_irq(&zone->lru_lock);
-			zone = pagezone;
-			spin_lock_irq(&zone->lru_lock);
-		}
-		BUG_ON(PageLRU(page));
-		SetPageLRU(page);
-		BUG_ON(PageActive(page));
-		SetPageActive(page);
-		add_page_to_active_list(zone, page);
-	}
-	if (zone)
-		spin_unlock_irq(&zone->lru_lock);
-	release_pages(pvec->pages, pvec->nr, pvec->cold);
-	pagevec_reinit(pvec);
-}
-
 static inline void lru_cache_add(struct page *page)
 {
 	struct pagevec *pvec = &get_cpu_var(lru_add_pvecs);

@@ -85,18 +27,16 @@ static inline void lru_cache_add_active(
 
 	page_cache_get(page);
 	if (!pagevec_add(pvec, page))
-		__pagevec_lru_add_active(pvec);
+		__pagevec_pgrep_add(pvec);
 	put_cpu_var(lru_add_active_pvecs);
 }
 
 void fastcall pgrep_add(struct page *page)
 {
-	if (PageActive(page)) {
-		ClearPageActive(page);
+	if (PageActive(page))
 		lru_cache_add_active(page);
-	} else {
+	else
 		lru_cache_add(page);
-	}
 }
 
 void __pgrep_add_drain(unsigned int cpu)

@@ -107,5 +47,5 @@ void __pgrep_add_drain(unsigned int cpu)
 		__pagevec_pgrep_add(pvec);
 	pvec = &per_cpu(lru_add_active_pvecs, cpu);
 	if (pagevec_count(pvec))
-		__pagevec_lru_add_active(pvec);
+		__pagevec_pgrep_add(pvec);
 }

Index: linux-2.6/mm/swap.c
===================================================================
@@ -335,6 +335,37 @@ unsigned pagevec_lookup_tag(struct pagev
 
 EXPORT_SYMBOL(pagevec_lookup_tag);
 
+/*
+ * Add the passed pages to the LRU, then drop the caller's refcount
+ * on them.  Reinitialises the caller's pagevec.
+ */
+void __pagevec_pgrep_add(struct pagevec *pvec)
+{
+	int i;
+	struct zone *zone = NULL;
+
+	for (i = 0; i < pagevec_count(pvec); i++) {
+		struct page *page = pvec->pages[i];
+		struct zone *pagezone = page_zone(page);
+
+		if (pagezone != zone) {
+			if (zone)
+				spin_unlock_irq(&zone->lru_lock);
+			zone = pagezone;
+			spin_lock_irq(&zone->lru_lock);
+		}
+		BUG_ON(PageLRU(page));
+		SetPageLRU(page);
+		__pgrep_add(zone, page);
+	}
+	if (zone)
+		spin_unlock_irq(&zone->lru_lock);
+	release_pages(pvec->pages, pvec->nr, pvec->cold);
+	pagevec_reinit(pvec);
+}
+
+EXPORT_SYMBOL(__pagevec_pgrep_add);
+
 #ifdef CONFIG_SMP
 /*
  * We tolerate a little inaccuracy to avoid ping-ponging the counter between

Index: linux-2.6/include/linux/mm_inline.h
===================================================================
@@ -1,3 +1,7 @@
+#ifndef _LINUX_MM_INLINE_H_
+#define _LINUX_MM_INLINE_H_
+
+#ifdef __KERNEL__
 
 static inline void
 add_page_to_active_list(struct zone *zone, struct page *page)

@@ -39,3 +43,5 @@ del_page_from_lru(struct zone *zone, str
 	}
 }
 
+#endif /* __KERNEL__ */
+#endif /* _LINUX_MM_INLINE_H_ */

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -3,6 +3,9 @@
 
 #ifdef __KERNEL__
 
+#include <linux/fs.h>
+#include <linux/rmap.h>
+
 static inline void pgrep_hint_active(struct page *page)
 {
 	SetPageActive(page);

@@ -21,5 +24,50 @@ __pgrep_add(struct zone *zone, struct pa
 		add_page_to_inactive_list(zone, page);
 }
 
+/* Called without lock on whether page is mapped, so answer is unstable */
+static inline int page_mapping_inuse(struct page *page)
+{
+	struct address_space *mapping;
+
+	/* Page is in somebody's page tables. */
+	if (page_mapped(page))
+		return 1;
+
+	/* Be more reluctant to reclaim swapcache than pagecache */
+	if (PageSwapCache(page))
+		return 1;
+
+	mapping = page_mapping(page);
+	if (!mapping)
+		return 0;
+
+	/* File is mmap'd by somebody? */
+	return mapping_mapped(mapping);
+}
+
+static inline reclaim_t pgrep_reclaimable(struct page *page)
+{
+	int referenced;
+
+	if (PageActive(page))
+		BUG();
+
+	referenced = page_referenced(page, 1);
+	/* In active use or really unfreeable?  Activate it. */
+	if (referenced && page_mapping_inuse(page))
+		return RECLAIM_ACTIVATE;
+
+	if (referenced)
+		return RECLAIM_REFERENCED;
+
+	return RECLAIM_OK;
+}
+
+static inline int pgrep_activate(struct page *page)
+{
+	SetPageActive(page);
+	return 1;
+}
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_USEONCE_POLICY_H */

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -17,6 +17,17 @@ extern void __pgrep_add_drain(unsigned i
 extern int pgrep_add_drain_all(void);
 extern void __pagevec_pgrep_add(struct pagevec *);
 
+typedef enum {
+	RECLAIM_KEEP,
+	RECLAIM_ACTIVATE,
+	RECLAIM_REFERENCED,
+	RECLAIM_OK,
+} reclaim_t;
+
+/* reclaim_t pgrep_reclaimable(struct page *); */
+/* int pgrep_activate(struct page *page); */
+
+
 #ifdef CONFIG_MM_POLICY_USEONCE
 #include <linux/mm_use_once_policy.h>
 #else

73 lines not on this page · raw message

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -8,6 +8,36 @@
 #include <linux/pagevec.h>
 #include <linux/mm_inline.h>
 
+#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
+
+#ifdef ARCH_HAS_PREFETCH
+#define prefetch_prev_lru_page(_page, _base, _field)			\
+	do {								\
+		if ((_page)->lru.prev != _base) {			\
+			struct page *prev;				\
+									\
+			prev = lru_to_page(&(_page->lru));		\
+			prefetch(&prev->_field);			\
+		}							\
+	} while (0)
+#else
+#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
+#endif
+
+#ifdef ARCH_HAS_PREFETCHW
+#define prefetchw_prev_lru_page(_page, _base, _field)			\
+	do {								\
+		if ((_page)->lru.prev != _base) {			\
+			struct page *prev;				\
+									\
+			prev = lru_to_page(&(_page->lru));		\
+			prefetchw(&prev->_field);			\
+		}							\
+	} while (0)
+#else
+#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
+#endif
+
 /* void pgrep_hint_active(struct page *); */
 /* void pgrep_hint_use_once(struct page *); */
 extern void fastcall pgrep_add(struct page *);

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -77,36 +77,6 @@ struct shrinker {
 	long			nr;	/* objs pending delete */
 };
 
-#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
-
-#ifdef ARCH_HAS_PREFETCH
-#define prefetch_prev_lru_page(_page, _base, _field)			\
-	do {								\
-		if ((_page)->lru.prev != _base) {			\
-			struct page *prev;				\
-									\
-			prev = lru_to_page(&(_page->lru));		\
-			prefetch(&prev->_field);			\
-		}							\
-	} while (0)
-#else
-#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
-#endif
-
-#ifdef ARCH_HAS_PREFETCHW
-#define prefetchw_prev_lru_page(_page, _base, _field)			\
-	do {								\
-		if ((_page)->lru.prev != _base) {			\
-			struct page *prev;				\
-									\
-			prev = lru_to_page(&(_page->lru));		\
-			prefetchw(&prev->_field);			\
-		}							\
-	} while (0)
-#else
-#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
-#endif
-
 /*
  * From 0 .. 100.  Higher means more swappy.
  */

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -8,6 +8,29 @@
 #include <linux/pagevec.h>
 #include <linux/mm_inline.h>
 
+struct scan_control {
+	/* Incremented by the number of inactive pages that were scanned */
+	unsigned long nr_scanned;
+
+	unsigned long nr_mapped;	/* From page_state */
+
+	/* This context's GFP mask */
+	gfp_t gfp_mask;
+
+	int may_writepage;
+
+	/* Can pages be swapped as part of reclaim? */
+	int may_swap;
+
+	/* This context's SWAP_CLUSTER_MAX. If freeing memory for
+	 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
+	 * In this context, it doesn't matter that we scan the
+	 * whole list at once. */
+	int swap_cluster_max;
+
+	unsigned long nr_writeout;      /* page against which writeout was started */
+};
+
 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
 
 #ifdef ARCH_HAS_PREFETCH

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -43,29 +43,6 @@
 
 #include "internal.h"
 
-struct scan_control {
-	/* Incremented by the number of inactive pages that were scanned */
-	unsigned long nr_scanned;
-
-	unsigned long nr_mapped;	/* From page_state */
-
-	/* This context's GFP mask */
-	gfp_t gfp_mask;
-
-	int may_writepage;
-
-	/* Can pages be swapped as part of reclaim? */
-	int may_swap;
-
-	/* This context's SWAP_CLUSTER_MAX. If freeing memory for
-	 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
-	 * In this context, it doesn't matter that we scan the
-	 * whole list at once. */
-	int swap_cluster_max;
-
-	unsigned long nr_writeout;	/* page against which writeout was started */
-};
-
 /*
  * The list of shrinker callbacks used by to apply pressure to
  * ageable caches.

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -618,6 +618,50 @@ done:
 	return nr_reclaimed;
 }
 
+int should_reclaim_mapped(struct zone *zone)
+{
+	long mapped_ratio;
+	long distress;
+	long swap_tendency;
+
+	/*
+	 * `distress' is a measure of how much trouble we're having
+	 * reclaiming pages.  0 -> no problems.  100 -> great trouble.
+	 */
+	distress = 100 >> zone->prev_priority;
+
+	/*
+	 * The point of this algorithm is to decide when to start
+	 * reclaiming mapped memory instead of just pagecache.  Work out
+	 * how much memory
+	 * is mapped.
+	 */
+	mapped_ratio = (read_page_state(nr_mapped) * 100) / total_memory;
+
+	/*
+	 * Now decide how much we really want to unmap some pages.  The
+	 * mapped ratio is downgraded - just because there's a lot of
+	 * mapped memory doesn't necessarily mean that page reclaim
+	 * isn't succeeding.
+	 *
+	 * The distress ratio is important - we don't want to start
+	 * going oom.
+	 *
+	 * A 100% value of vm_swappiness overrides this algorithm
+	 * altogether.
+	 */
+	swap_tendency = mapped_ratio / 2 + distress + vm_swappiness;
+
+	/*
+	 * Now use this metric to decide whether to start moving mapped
+	 * memory onto the inactive list.
+	 */
+	if (swap_tendency >= 100)
+		return 1;
+
+	return 0;
+}
+
 /*
  * This moves pages from the active list to the inactive list.
  *

@@ -636,7 +680,7 @@ done:
  * But we had to alter page->flags anyway.
  */
 static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
-				struct scan_control *sc)
+				struct scan_control *sc, int reclaim_mapped)
 {
 	unsigned long pgmoved;
 	int pgdeactivate = 0;

@@ -646,48 +690,9 @@ static void shrink_active_list(unsigned 
 	LIST_HEAD(l_active);	/* Pages to go onto the active_list */
 	struct page *page;
 	struct pagevec pvec;
-	int reclaim_mapped = 0;
-
-	if (sc->may_swap) {
-		long mapped_ratio;
-		long distress;
-		long swap_tendency;
-
-		/*
-		 * `distress' is a measure of how much trouble we're having
-		 * reclaiming pages.  0 -> no problems.  100 -> great trouble.
-		 */
-		distress = 100 >> zone->prev_priority;
-
-		/*
-		 * The point of this algorithm is to decide when to start
-		 * reclaiming mapped memory instead of just pagecache.  Work out
-		 * how much memory
-		 * is mapped.
-		 */
-		mapped_ratio = (sc->nr_mapped * 100) / total_memory;
 
-		/*
-		 * Now decide how much we really want to unmap some pages.  The
-		 * mapped ratio is downgraded - just because there's a lot of
-		 * mapped memory doesn't necessarily mean that page reclaim
-		 * isn't succeeding.
-		 *
-		 * The distress ratio is important - we don't want to start
-		 * going oom.
-		 *
-		 * A 100% value of vm_swappiness overrides this algorithm
-		 * altogether.
-		 */
-		swap_tendency = mapped_ratio / 2 + distress + vm_swappiness;
-
-		/*
-		 * Now use this metric to decide whether to start moving mapped
-		 * memory onto the inactive list.
-		 */
-		if (swap_tendency >= 100)
-			reclaim_mapped = 1;
-	}
+	if (!sc->may_swap)
+		reclaim_mapped = 0;
 
 	pgrep_add_drain();
 	spin_lock_irq(&zone->lru_lock);

@@ -781,6 +786,7 @@ static unsigned long shrink_zone(int pri
 	unsigned long nr_inactive;
 	unsigned long nr_to_scan;
 	unsigned long nr_reclaimed = 0;
+	int reclaim_mapped = should_reclaim_mapped(zone);
 
 	atomic_inc(&zone->reclaim_in_progress);
 

@@ -807,7 +813,7 @@ static unsigned long shrink_zone(int pri
 			nr_to_scan = min(nr_active,
 					(unsigned long)sc->swap_cluster_max);
 			nr_active -= nr_to_scan;
-			shrink_active_list(nr_to_scan, zone, sc);
+			shrink_active_list(nr_to_scan, zone, sc, reclaim_mapped);
 		}
 
 		if (nr_inactive) {

@@ -910,7 +916,6 @@ unsigned long try_to_free_pages(struct z
 	}
 
 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
-		sc.nr_mapped = read_page_state(nr_mapped);
 		sc.nr_scanned = 0;
 		if (!priority)
 			disable_swap_token();

@@ -1000,7 +1005,6 @@ loop_again:
 	total_scanned = 0;
 	nr_reclaimed = 0;
 	sc.may_writepage = !laptop_mode;
-	sc.nr_mapped = read_page_state(nr_mapped);
 
 	inc_page_state(pageoutrun);
 

@@ -1351,7 +1355,6 @@ static int __zone_reclaim(struct zone *z
 	struct scan_control sc = {
 		.may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
 		.may_swap = !!(zone_reclaim_mode & RECLAIM_SWAP),
-		.nr_mapped = read_page_state(nr_mapped),
 		.swap_cluster_max = max_t(unsigned long, nr_pages,
 					SWAP_CLUSTER_MAX),
 		.gfp_mask = gfp_mask,

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -12,8 +12,6 @@ struct scan_control {
 	/* Incremented by the number of inactive pages that were scanned */
 	unsigned long nr_scanned;
 
-	unsigned long nr_mapped;	/* From page_state */
-
 	/* This context's GFP mask */
 	gfp_t gfp_mask;
 

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -24,6 +24,32 @@ __pgrep_add(struct zone *zone, struct pa
 		add_page_to_inactive_list(zone, page);
 }
 
+/*
+ * Mark a page as having seen activity.
+ *
+ * inactive,unreferenced	->	inactive,referenced
+ * inactive,referenced		->	active,unreferenced
+ * active,unreferenced		->	active,referenced
+ */
+static inline void pgrep_mark_accessed(struct page *page)
+{
+	if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
+		struct zone *zone = page_zone(page);
+
+		spin_lock_irq(&zone->lru_lock);
+		if (PageLRU(page) && !PageActive(page)) {
+			del_page_from_inactive_list(zone, page);
+			SetPageActive(page);
+			add_page_to_active_list(zone, page);
+			inc_page_state(pgactivate);
+		}
+		spin_unlock_irq(&zone->lru_lock);
+		ClearPageReferenced(page);
+	} else if (!PageReferenced(page)) {
+		SetPageReferenced(page);
+	}
+}
+
 /* Called without lock on whether page is mapped, so answer is unstable */
 static inline int page_mapping_inuse(struct page *page)
 {

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -77,6 +77,7 @@ typedef enum {
 
 /* reclaim_t pgrep_reclaimable(struct page *); */
 /* int pgrep_activate(struct page *page); */
+/* void pgrep_mark_accessed(struct page *); */
 
 
 #ifdef CONFIG_MM_POLICY_USEONCE

Index: linux-2.6/mm/swap.c
===================================================================
@@ -98,37 +98,11 @@ int rotate_reclaimable_page(struct page 
 }
 
 /*
- * FIXME: speed this up?
- */
-void fastcall activate_page(struct page *page)
-{
-	struct zone *zone = page_zone(page);
-
-	spin_lock_irq(&zone->lru_lock);
-	if (PageLRU(page) && !PageActive(page)) {
-		del_page_from_inactive_list(zone, page);
-		SetPageActive(page);
-		add_page_to_active_list(zone, page);
-		inc_page_state(pgactivate);
-	}
-	spin_unlock_irq(&zone->lru_lock);
-}
-
-/*
  * Mark a page as having seen activity.
- *
- * inactive,unreferenced	->	inactive,referenced
- * inactive,referenced		->	active,unreferenced
- * active,unreferenced		->	active,referenced
  */
 void fastcall mark_page_accessed(struct page *page)
 {
-	if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
-		activate_page(page);
-		ClearPageReferenced(page);
-	} else if (!PageReferenced(page)) {
-		SetPageReferenced(page);
-	}
+	pgrep_mark_accessed(page);
 }
 
 EXPORT_SYMBOL(mark_page_accessed);

Index: linux-2.6/include/linux/swap.h
===================================================================
@@ -7,6 +7,7 @@
 #include <linux/mmzone.h>
 #include <linux/list.h>
 #include <linux/sched.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/atomic.h>
 #include <asm/page.h>

@@ -170,10 +171,16 @@ extern void swap_setup(void);
 extern void release_pages(struct page **, int, int);
 
 /* linux/mm/vmscan.c */
+extern int remove_mapping(struct address_space *mapping, struct page *page);
+extern unsigned long shrink_page_list(struct list_head *page_list,
+					struct scan_control *sc);
+extern unsigned long isolate_lru_pages(unsigned long nr_to_scan,
+		struct list_head *src, struct list_head *dst,
+		unsigned long *scanned);
+extern int should_reclaim_mapped(struct zone *zone);
 extern unsigned long try_to_free_pages(struct zone **, gfp_t);
 extern unsigned long shrink_all_memory(unsigned long nr_pages);
 extern int vm_swappiness;
-extern int remove_mapping(struct address_space *mapping, struct page *page);
 
 /* possible outcome of pageout() */
 typedef enum {

252 lines not on this page · raw message

293 lines not on this page · raw message

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -78,6 +78,8 @@ typedef enum {
 /* reclaim_t pgrep_reclaimable(struct page *); */
 /* int pgrep_activate(struct page *page); */
 /* void pgrep_mark_accessed(struct page *); */
+extern unsigned long pgrep_shrink_zone(int priority, struct zone *zone,
+					struct scan_control *sc);
 
 
 #ifdef CONFIG_MM_POLICY_USEONCE

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -95,5 +95,13 @@ static inline int pgrep_activate(struct 
 	return 1;
 }
 
+static inline void __pgrep_rotate_reclaimable(struct zone *zone, struct page *page)
+{
+	if (PageLRU(page) && !PageActive(page)) {
+		list_move_tail(&page->lru, &zone->inactive_list);
+		inc_page_state(pgrotated);
+	}
+}
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_USEONCE_POLICY_H */

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -80,6 +80,7 @@ typedef enum {
 /* void pgrep_mark_accessed(struct page *); */
 extern unsigned long pgrep_shrink_zone(int priority, struct zone *zone,
 					struct scan_control *sc);
+/* void __pgrep_rotate_reclaimable(struct zone *, struct page *); */
 
 
 #ifdef CONFIG_MM_POLICY_USEONCE

Index: linux-2.6/mm/swap.c
===================================================================
@@ -79,18 +79,12 @@ int rotate_reclaimable_page(struct page 
 		return 1;
 	if (PageDirty(page))
 		return 1;
-	if (PageActive(page))
-		return 1;
 	if (!PageLRU(page))
 		return 1;
 
 	zone = page_zone(page);
 	spin_lock_irqsave(&zone->lru_lock, flags);
-	if (PageLRU(page) && !PageActive(page)) {
-		list_del(&page->lru);
-		list_add_tail(&page->lru, &zone->inactive_list);
-		inc_page_state(pgrotated);
-	}
+	__pgrep_rotate_reclaimable(zone, page);
 	if (!test_clear_page_writeback(page))
 		BUG();
 	spin_unlock_irqrestore(&zone->lru_lock, flags);

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -81,6 +81,9 @@ typedef enum {
 extern unsigned long pgrep_shrink_zone(int priority, struct zone *zone,
 					struct scan_control *sc);
 /* void __pgrep_rotate_reclaimable(struct zone *, struct page *); */
+/* void pgrep_copy_state(struct page *, struct page *); */
+/* void pgrep_clear_state(struct page *); */
+/* int pgrep_is_active(struct page *); */
 
 
 #ifdef CONFIG_MM_POLICY_USEONCE

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -103,5 +103,22 @@ static inline void __pgrep_rotate_reclai
 	}
 }
 
+static inline void pgrep_copy_state(struct page *dpage, struct page *spage)
+{
+	if (PageActive(spage))
+		SetPageActive(dpage);
+}
+
+static inline void pgrep_clear_state(struct page *page)
+{
+	if (PageActive(page))
+		ClearPageActive(page);
+}
+
+static inline int pgrep_is_active(struct page *page)
+{
+	return PageActive(page);
+}
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_USEONCE_POLICY_H */

Index: linux-2.6/mm/mempolicy.c
===================================================================
@@ -1749,7 +1749,7 @@ static void gather_stats(struct page *pa
 	if (PageSwapCache(page))
 		md->swapcache++;
 
-	if (PageActive(page))
+	if (pgrep_is_active(page))
 		md->active++;
 
 	if (PageWriteback(page))

Index: linux-2.6/mm/migrate.c
===================================================================
@@ -262,12 +262,11 @@ void migrate_page_copy(struct page *newp
 		SetPageReferenced(newpage);
 	if (PageUptodate(page))
 		SetPageUptodate(newpage);
-	if (PageActive(page))
-		SetPageActive(newpage);
 	if (PageChecked(page))
 		SetPageChecked(newpage);
 	if (PageMappedToDisk(page))
 		SetPageMappedToDisk(newpage);
+	pgrep_copy_state(newpage, page);
 
 	if (PageDirty(page)) {
 		clear_page_dirty_for_io(page);

@@ -275,8 +274,8 @@ void migrate_page_copy(struct page *newp
  	}
 
 	ClearPageSwapCache(page);
-	ClearPageActive(page);
 	ClearPagePrivate(page);
+	pgrep_clear_state(page);
 	set_page_private(page, 0);
 	page->mapping = NULL;
 

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -473,6 +473,7 @@ unsigned long shrink_page_list(struct li
 			goto keep_locked;
 
 free_it:
+		pgrep_clear_state(page);
 		unlock_page(page);
 		nr_reclaimed++;
 		if (!pagevec_add(&freed_pvec, page))

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -84,6 +84,7 @@ extern unsigned long pgrep_shrink_zone(i
 /* void pgrep_copy_state(struct page *, struct page *); */
 /* void pgrep_clear_state(struct page *); */
 /* int pgrep_is_active(struct page *); */
+/* void __pgrep_remove(struct zone *zone, struct page *page); */
 
 
 #ifdef CONFIG_MM_POLICY_USEONCE

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -120,5 +120,14 @@ static inline int pgrep_is_active(struct
 	return PageActive(page);
 }
 
+static inline void __pgrep_remove(struct zone *zone, struct page *page)
+{
+	list_del(&page->lru);
+	if (PageActive(page))
+		zone->nr_active--;
+	else
+		zone->nr_inactive--;
+}
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_USEONCE_POLICY_H */

Index: linux-2.6/mm/migrate.c
===================================================================
@@ -53,10 +53,7 @@ int isolate_lru_page(struct page *page, 
 			ret = 0;
 			get_page(page);
 			ClearPageLRU(page);
-			if (PageActive(page))
-				del_page_from_active_list(zone, page);
-			else
-				del_page_from_inactive_list(zone, page);
+			__pgrep_remove(zone, page);
 			list_add_tail(&page->lru, pagelist);
 		}
 		spin_unlock_irq(&zone->lru_lock);

Index: linux-2.6/mm/swap.c
===================================================================
@@ -140,7 +140,8 @@ void fastcall __page_cache_release(struc
 		spin_lock_irqsave(&zone->lru_lock, flags);
 		BUG_ON(!PageLRU(page));
 		__ClearPageLRU(page);
-		del_page_from_lru(zone, page);
+		__pgrep_remove(zone, page);
+		pgrep_clear_state(page);
 		spin_unlock_irqrestore(&zone->lru_lock, flags);
 	}
 	free_hot_page(page);

@@ -191,7 +192,8 @@ void release_pages(struct page **pages, 
 			}
 			BUG_ON(!PageLRU(page));
 			__ClearPageLRU(page);
-			del_page_from_lru(zone, page);
+			__pgrep_remove(zone, page);
+			pgrep_clear_state(page);
 		}
 
 		if (!pagevec_add(&pages_to_free, page)) {

Index: linux-2.6/mm/useonce.c
===================================================================
@@ -73,11 +73,9 @@ static unsigned long shrink_inactive_lis
 		unsigned long nr_scan;
 		unsigned long nr_freed;
 
-		nr_taken = isolate_lru_pages(sc->swap_cluster_max,
+		nr_taken = isolate_lru_pages(zone, sc->swap_cluster_max,
 					     &zone->inactive_list,
 					     &page_list, &nr_scan);
-		zone->nr_inactive -= nr_taken;
-		zone->pages_scanned += nr_scan;
 		spin_unlock_irq(&zone->lru_lock);
 
 		nr_scanned += nr_scan;

@@ -155,10 +153,8 @@ static void shrink_active_list(unsigned 
 
 	pgrep_add_drain();
 	spin_lock_irq(&zone->lru_lock);
-	pgmoved = isolate_lru_pages(nr_pages, &zone->active_list,
+	pgmoved = isolate_lru_pages(zone, nr_pages, &zone->active_list,
 				    &l_hold, &pgscanned);
-	zone->pages_scanned += pgscanned;
-	zone->nr_active -= pgmoved;
 	spin_unlock_irq(&zone->lru_lock);
 
 	while (!list_empty(&l_hold)) {

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -514,7 +514,7 @@ keep:
  *
  * returns how many pages were moved onto *@dst.
  */
-unsigned long isolate_lru_pages(unsigned long nr_to_scan,
+unsigned long isolate_lru_pages(struct zone *zone, unsigned long nr_to_scan,
 		struct list_head *src, struct list_head *dst,
 		unsigned long *scanned)
 {

@@ -523,14 +523,11 @@ unsigned long isolate_lru_pages(unsigned
 	unsigned long scan;
 
 	for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
-		struct list_head *target;
 		page = lru_to_page(src);
 		prefetchw_prev_lru_page(page, src, flags);
 
 		BUG_ON(!PageLRU(page));
 
-		list_del(&page->lru);
-		target = src;
 		if (likely(get_page_unless_zero(page))) {
 			/*
 			 * Be careful not to clear PageLRU until after we're

@@ -538,14 +535,15 @@ unsigned long isolate_lru_pages(unsigned
 			 * page release code relies on it.
 			 */
 			ClearPageLRU(page);
-			target = dst;
+			__pgrep_remove(zone, page);
+			list_add(&page->lru, dst);
 			nr_taken++;
 		} /* else it is being freed elsewhere */
-
-		list_add(&page->lru, target);
+		else list_move(&page->lru, src);
 	}
 
 	*scanned = scan;
+	zone->pages_scanned += scan;
 	return nr_taken;
 }
 

Index: linux-2.6/include/linux/swap.h
===================================================================
@@ -174,9 +174,9 @@ extern void release_pages(struct page **
 extern int remove_mapping(struct address_space *mapping, struct page *page);
 extern unsigned long shrink_page_list(struct list_head *page_list,
 					struct scan_control *sc);
-extern unsigned long isolate_lru_pages(unsigned long nr_to_scan,
-		struct list_head *src, struct list_head *dst,
-		unsigned long *scanned);
+extern unsigned long isolate_lru_pages(struct zone *zone,
+		unsigned long nr_to_scan, struct list_head *src,
+		struct list_head *dst, unsigned long *scanned);
 extern int should_reclaim_mapped(struct zone *zone);
 extern unsigned long try_to_free_pages(struct zone **, gfp_t);
 extern unsigned long shrink_all_memory(unsigned long nr_pages);

Index: linux-2.6/include/linux/mm_inline.h
===================================================================
@@ -1,47 +0,0 @@
-#ifndef _LINUX_MM_INLINE_H_
-#define _LINUX_MM_INLINE_H_
-
-#ifdef __KERNEL__
-
-static inline void
-add_page_to_active_list(struct zone *zone, struct page *page)
-{
-	list_add(&page->lru, &zone->active_list);
-	zone->nr_active++;
-}
-
-static inline void
-add_page_to_inactive_list(struct zone *zone, struct page *page)
-{
-	list_add(&page->lru, &zone->inactive_list);
-	zone->nr_inactive++;
-}
-
-static inline void
-del_page_from_active_list(struct zone *zone, struct page *page)
-{
-	list_del(&page->lru);
-	zone->nr_active--;
-}
-
-static inline void
-del_page_from_inactive_list(struct zone *zone, struct page *page)
-{
-	list_del(&page->lru);
-	zone->nr_inactive--;
-}
-
-static inline void
-del_page_from_lru(struct zone *zone, struct page *page)
-{
-	list_del(&page->lru);
-	if (PageActive(page)) {
-		__ClearPageActive(page);
-		zone->nr_active--;
-	} else {
-		zone->nr_inactive--;
-	}
-}
-
-#endif /* __KERNEL__ */
-#endif /* _LINUX_MM_INLINE_H_ */

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -6,7 +6,6 @@
 #include <linux/mmzone.h>
 #include <linux/mm.h>
 #include <linux/pagevec.h>
-#include <linux/mm_inline.h>
 
 struct scan_control {
 	/* Incremented by the number of inactive pages that were scanned */

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -6,6 +6,34 @@
 #include <linux/fs.h>
 #include <linux/rmap.h>
 
+static inline void
+add_page_to_active_list(struct zone *zone, struct page *page)
+{
+	list_add(&page->lru, &zone->active_list);
+	zone->nr_active++;
+}
+
+static inline void
+add_page_to_inactive_list(struct zone *zone, struct page *page)
+{
+	list_add(&page->lru, &zone->inactive_list);
+	zone->nr_inactive++;
+}
+
+static inline void
+del_page_from_active_list(struct zone *zone, struct page *page)
+{
+	list_del(&page->lru);
+	zone->nr_active--;
+}
+
+static inline void
+del_page_from_inactive_list(struct zone *zone, struct page *page)
+{
+	list_del(&page->lru);
+	zone->nr_inactive--;
+}
+
 static inline void pgrep_hint_active(struct page *page)
 {
 	SetPageActive(page);

Index: linux-2.6/mm/migrate.c
===================================================================
@@ -17,7 +17,6 @@
 #include <linux/swap.h>
 #include <linux/pagemap.h>
 #include <linux/buffer_head.h>
-#include <linux/mm_inline.h>
 #include <linux/pagevec.h>
 #include <linux/rmap.h>
 #include <linux/topology.h>

Index: linux-2.6/mm/swap.c
===================================================================
@@ -22,7 +22,6 @@
 #include <linux/pagevec.h>
 #include <linux/init.h>
 #include <linux/module.h>
-#include <linux/mm_inline.h>
 #include <linux/buffer_head.h>	/* for try_to_release_page() */
 #include <linux/module.h>
 #include <linux/percpu_counter.h>

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -24,7 +24,6 @@
 #include <linux/blkdev.h>
 #include <linux/buffer_head.h>	/* for try_to_release_page(),
 					buffer_heads_over_limit */
-#include <linux/mm_inline.h>
 #include <linux/pagevec.h>
 #include <linux/backing-dev.h>
 #include <linux/rmap.h>

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -84,7 +84,7 @@ extern unsigned long pgrep_shrink_zone(i
 /* void pgrep_clear_state(struct page *); */
 /* int pgrep_is_active(struct page *); */
 /* void __pgrep_remove(struct zone *zone, struct page *page); */
-
+extern void pgrep_reinsert(struct list_head *);
 
 #ifdef CONFIG_MM_POLICY_USEONCE
 #include <linux/mm_use_once_policy.h>

Index: linux-2.6/mm/useonce.c
===================================================================
@@ -52,6 +52,16 @@ void __pgrep_add_drain(unsigned int cpu)
 		__pagevec_pgrep_add(pvec);
 }
 
+void pgrep_reinsert(struct list_head *page_list)
+{
+	struct page *page, *page2;
+
+	list_for_each_entry_safe(page, page2, page_list, lru) {
+		list_del(&page->lru);
+		pgrep_add(page);
+		put_page(page);
+	}
+}
 /*
  * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
  * of reclaimed pages

Index: linux-2.6/mm/mempolicy.c
===================================================================
@@ -607,7 +607,7 @@ int migrate_to_node(struct mm_struct *mm
 	if (!list_empty(&pagelist)) {
 		err = migrate_pages_to(&pagelist, NULL, dest);
 		if (!list_empty(&pagelist))
-			putback_lru_pages(&pagelist);
+			pgrep_reinsert(&pagelist);
 	}
 	return err;
 }

@@ -775,7 +775,7 @@ long do_mbind(unsigned long start, unsig
 	}
 
 	if (!list_empty(&pagelist))
-		putback_lru_pages(&pagelist);
+		pgrep_reinsert(&pagelist);
 
 	up_write(&mm->mmap_sem);
 	mpol_free(new);

Index: linux-2.6/include/linux/migrate.h
===================================================================
@@ -6,7 +6,6 @@
 
 #ifdef CONFIG_MIGRATION
 extern int isolate_lru_page(struct page *p, struct list_head *pagelist);
-extern int putback_lru_pages(struct list_head *l);
 extern int migrate_page(struct page *, struct page *);
 extern void migrate_page_copy(struct page *, struct page *);
 extern int migrate_page_remove_references(struct page *, struct page *, int);

@@ -22,7 +21,6 @@ extern int migrate_prep(void);
 
 static inline int isolate_lru_page(struct page *p, struct list_head *list)
 					{ return -ENOSYS; }
-static inline int putback_lru_pages(struct list_head *l) { return 0; }
 static inline int migrate_pages(struct list_head *l, struct list_head *t,
 	struct list_head *moved, struct list_head *failed) { return -ENOSYS; }
 

Index: linux-2.6/mm/migrate.c
===================================================================
@@ -25,8 +25,6 @@
 #include <linux/swapops.h>
 #include <linux/mm_page_replace.h>
 
-#include "internal.h"
-
 /* The maximum number of pages to take off the LRU for migration */
 #define MIGRATE_CHUNK_SIZE 256
 

@@ -82,31 +80,6 @@ int migrate_prep(void)
 	return 0;
 }
 
-static inline void move_to_lru(struct page *page)
-{
-	list_del(&page->lru);
-	pgrep_add(page);
-	put_page(page);
-}
-
-/*
- * Add isolated pages on the list back to the LRU.
- *
- * returns the number of pages put back.
- */
-int putback_lru_pages(struct list_head *l)
-{
-	struct page *page;
-	struct page *page2;
-	int count = 0;
-
-	list_for_each_entry_safe(page, page2, l, lru) {
-		move_to_lru(page);
-		count++;
-	}
-	return count;
-}
-
 /*
  * Non migratable page
  */

@@ -626,7 +599,7 @@ redo:
 	}
 	err = migrate_pages(pagelist, &newlist, &moved, &failed);
 
-	putback_lru_pages(&moved);	/* Call release pages instead ?? */
+	pgrep_reinsert(&moved);	/* Call release pages instead ?? */
 
 	if (err >= 0 && list_empty(&newlist) && !list_empty(pagelist))
 		goto redo;

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -58,6 +58,8 @@ struct scan_control {
 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
 #endif
 
+extern void pgrep_init(void);
+extern void pgrep_init_zone(struct zone *);
 /* void pgrep_hint_active(struct page *); */
 /* void pgrep_hint_use_once(struct page *); */
 extern void fastcall pgrep_add(struct page *);

Index: linux-2.6/mm/useonce.c
===================================================================
@@ -6,6 +6,21 @@
 #include <linux/buffer_head.h> /* for try_to_release_page(),
                                        buffer_heads_over_limit */
 
+void __init pgrep_init(void)
+{
+	/* empty hook */
+}
+
+void __init pgrep_init_zone(struct zone *zone)
+{
+	INIT_LIST_HEAD(&zone->active_list);
+	INIT_LIST_HEAD(&zone->inactive_list);
+	zone->nr_scan_active = 0;
+	zone->nr_scan_inactive = 0;
+	zone->nr_active = 0;
+	zone->nr_inactive = 0;
+}
+
 /**
  * lru_cache_add: add a page to the page lists
  * @page: the page to add

Index: linux-2.6/mm/page_alloc.c
===================================================================
@@ -37,6 +37,7 @@
 #include <linux/nodemask.h>
 #include <linux/vmalloc.h>
 #include <linux/mempolicy.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/tlbflush.h>
 #include <asm/div64.h>

@@ -2100,12 +2101,7 @@ static void __init free_area_init_core(s
 		zone->temp_priority = zone->prev_priority = DEF_PRIORITY;
 
 		zone_pcp_init(zone);
-		INIT_LIST_HEAD(&zone->active_list);
-		INIT_LIST_HEAD(&zone->inactive_list);
-		zone->nr_scan_active = 0;
-		zone->nr_scan_inactive = 0;
-		zone->nr_active = 0;
-		zone->nr_inactive = 0;
+		pgrep_init_zone(zone);
 		atomic_set(&zone->reclaim_in_progress, 0);
 		if (!size)
 			continue;

Index: linux-2.6/init/main.c
===================================================================
@@ -47,6 +47,7 @@
 #include <linux/rmap.h>
 #include <linux/mempolicy.h>
 #include <linux/key.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/io.h>
 #include <asm/bugs.h>

@@ -511,6 +512,7 @@ asmlinkage void __init start_kernel(void
 #endif
 	vfs_caches_init_early();
 	cpuset_init_early();
+	pgrep_init();
 	mem_init();
 	kmem_cache_init();
 	setup_per_cpu_pageset();

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -6,6 +6,7 @@
 #include <linux/mmzone.h>
 #include <linux/mm.h>
 #include <linux/pagevec.h>
+#include <linux/seq_file.h>
 
 struct scan_control {
 	/* Incremented by the number of inactive pages that were scanned */

@@ -87,6 +88,8 @@ extern unsigned long pgrep_shrink_zone(i
 /* int pgrep_is_active(struct page *); */
 /* void __pgrep_remove(struct zone *zone, struct page *page); */
 extern void pgrep_reinsert(struct list_head *);
+extern void pgrep_show(struct zone *);
+extern void pgrep_zoneinfo(struct zone *, struct seq_file *);
 
 #ifdef CONFIG_MM_POLICY_USEONCE
 #include <linux/mm_use_once_policy.h>

55 lines not on this page · raw message

56 lines not on this page · raw message

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -90,6 +90,9 @@ extern unsigned long pgrep_shrink_zone(i
 extern void pgrep_reinsert(struct list_head *);
 extern void pgrep_show(struct zone *);
 extern void pgrep_zoneinfo(struct zone *, struct seq_file *);
+extern void __pgrep_counts(unsigned long *, unsigned long *,
+				  unsigned long *, struct zone *);
+/* unsigned long __pgrep_nr_pages(struct zone *); */
 
 #ifdef CONFIG_MM_POLICY_USEONCE
 #include <linux/mm_use_once_policy.h>

Index: linux-2.6/mm/useonce.c
===================================================================
@@ -362,3 +362,18 @@ void pgrep_zoneinfo(struct zone *zone, s
 		   zone->spanned_pages,
 		   zone->present_pages);
 }
+
+void __pgrep_counts(unsigned long *active, unsigned long *inactive,
+			   unsigned long *free, struct zone *zones)
+{
+	int i;
+
+	*active = 0;
+	*inactive = 0;
+	*free = 0;
+	for (i = 0; i < MAX_NR_ZONES; i++) {
+		*active += zones[i].nr_active;
+		*inactive += zones[i].nr_inactive;
+		*free += zones[i].free_pages;
+	}
+}

Index: linux-2.6/mm/page_alloc.c
===================================================================
@@ -1332,17 +1332,7 @@ EXPORT_SYMBOL(mod_page_state_offset);
 void __get_zone_counts(unsigned long *active, unsigned long *inactive,
 			unsigned long *free, struct pglist_data *pgdat)
 {
-	struct zone *zones = pgdat->node_zones;
-	int i;
-
-	*active = 0;
-	*inactive = 0;
-	*free = 0;
-	for (i = 0; i < MAX_NR_ZONES; i++) {
-		*active += zones[i].nr_active;
-		*inactive += zones[i].nr_inactive;
-		*free += zones[i].free_pages;
-	}
+	__pgrep_counts(active, inactive, free, pgdat->node_zones);
 }
 
 void get_zone_counts(unsigned long *active,

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -157,5 +157,10 @@ static inline void __pgrep_remove(struct
 		zone->nr_inactive--;
 }
 
+static inline unsigned long __pgrep_nr_pages(struct zone *zone)
+{
+	return zone->nr_active + zone->nr_inactive;
+}
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_USEONCE_POLICY_H */

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -671,7 +671,7 @@ unsigned long try_to_free_pages(struct z
 			continue;
 
 		zone->temp_priority = DEF_PRIORITY;
-		lru_pages += zone->nr_active + zone->nr_inactive;
+		lru_pages += __pgrep_nr_pages(zone);
 	}
 
 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {

@@ -812,7 +812,7 @@ scan:
 		for (i = 0; i <= end_zone; i++) {
 			struct zone *zone = pgdat->node_zones + i;
 
-			lru_pages += zone->nr_active + zone->nr_inactive;
+			lru_pages += __pgrep_nr_pages(zone);
 		}
 
 		/*

@@ -853,7 +853,7 @@ scan:
 			if (zone->all_unreclaimable)
 				continue;
 			if (nr_slab == 0 && zone->pages_scanned >=
-				    (zone->nr_active + zone->nr_inactive) * 4)
+				    __pgrep_nr_pages(zone) * 4)
 				zone->all_unreclaimable = 1;
 			/*
 			 * If we've done a decent amount of scanning and

Index: linux-2.6/include/linux/mm_use_once_data.h
===================================================================
@@ -0,0 +1,16 @@
+#ifndef _LINUX_MM_USEONCE_DATA_H
+#define _LINUX_MM_USEONCE_DATA_H
+
+#ifdef __KERNEL__
+
+struct pgrep_data {
+	struct list_head	active_list;
+	struct list_head	inactive_list;
+	unsigned long		nr_scan_active;
+	unsigned long		nr_scan_inactive;
+	unsigned long		nr_active;
+	unsigned long		nr_inactive;
+};
+
+#endif /* __KERNEL__ */
+#endif /* _LINUX_MM_USEONCE_DATA_H */

130 lines not on this page · raw message

Index: linux-2.6/include/linux/mm_page_replace_data.h
===================================================================
@@ -0,0 +1,13 @@
+#ifndef _LINUX_MM_PAGE_REPLACE_DATA_H
+#define _LINUX_MM_PAGE_REPLACE_DATA_H
+
+#ifdef __KERNEL__
+
+#ifdef CONFIG_MM_POLICY_USEONCE
+#include <linux/mm_use_once_data.h>
+#else
+#error no mm policy
+#endif
+
+#endif /* __KERNEL__ */
+#endif /* _LINUX_MM_PAGE_REPLACE_DATA_H */

Index: linux-2.6/include/linux/mmzone.h
===================================================================
@@ -14,6 +14,7 @@
 #include <linux/init.h>
 #include <linux/seqlock.h>
 #include <linux/nodemask.h>
+#include <linux/mm_page_replace_data.h>
 #include <asm/atomic.h>
 #include <asm/page.h>
 

@@ -154,12 +155,7 @@ struct zone {
 
 	/* Fields commonly accessed by the page reclaim scanner */
 	spinlock_t		lru_lock;	
-	struct list_head	active_list;
-	struct list_head	inactive_list;
-	unsigned long		nr_scan_active;
-	unsigned long		nr_scan_inactive;
-	unsigned long		nr_active;
-	unsigned long		nr_inactive;
+	struct pgrep_data policy;
 	unsigned long		pages_scanned;	   /* since last reclaim */
 	int			all_unreclaimable; /* All pages pinned */
 

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -9,29 +9,29 @@
 static inline void
 add_page_to_active_list(struct zone *zone, struct page *page)
 {
-	list_add(&page->lru, &zone->active_list);
-	zone->nr_active++;
+	list_add(&page->lru, &zone->policy.active_list);
+	zone->policy.nr_active++;
 }
 
 static inline void
 add_page_to_inactive_list(struct zone *zone, struct page *page)
 {
-	list_add(&page->lru, &zone->inactive_list);
-	zone->nr_inactive++;
+	list_add(&page->lru, &zone->policy.inactive_list);
+	zone->policy.nr_inactive++;
 }
 
 static inline void
 del_page_from_active_list(struct zone *zone, struct page *page)
 {
 	list_del(&page->lru);
-	zone->nr_active--;
+	zone->policy.nr_active--;
 }
 
 static inline void
 del_page_from_inactive_list(struct zone *zone, struct page *page)
 {
 	list_del(&page->lru);
-	zone->nr_inactive--;
+       zone->policy.nr_inactive--;
 }
 
 static inline void pgrep_hint_active(struct page *page)

@@ -126,7 +126,7 @@ static inline int pgrep_activate(struct 
 static inline void __pgrep_rotate_reclaimable(struct zone *zone, struct page *page)
 {
 	if (PageLRU(page) && !PageActive(page)) {
-		list_move_tail(&page->lru, &zone->inactive_list);
+		list_move_tail(&page->lru, &zone->policy.inactive_list);
 		inc_page_state(pgrotated);
 	}
 }

@@ -152,14 +152,14 @@ static inline void __pgrep_remove(struct
 {
 	list_del(&page->lru);
 	if (PageActive(page))
-		zone->nr_active--;
+		zone->policy.nr_active--;
 	else
-		zone->nr_inactive--;
+		zone->policy.nr_inactive--;
 }
 
 static inline unsigned long __pgrep_nr_pages(struct zone *zone)
 {
-	return zone->nr_active + zone->nr_inactive;
+	return zone->policy.nr_active + zone->policy.nr_inactive;
 }
 
 #endif /* __KERNEL__ */

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -5,6 +5,14 @@
 
 #include <linux/fs.h>
 #include <linux/rmap.h>
+#include <linux/page-flags.h>
+
+#define PG_active	PG_reclaim1
+
+#define PageActive(page)	test_bit(PG_active, &(page)->flags)
+#define SetPageActive(page)	set_bit(PG_active, &(page)->flags)
+#define ClearPageActive(page)	clear_bit(PG_active, &(page)->flags)
+#define __ClearPageActive(page)	__clear_bit(PG_active, &(page)->flags)
 
 static inline void
 add_page_to_active_list(struct zone *zone, struct page *page)

Index: linux-2.6/include/linux/page-flags.h
===================================================================
@@ -70,7 +70,7 @@
 
 #define PG_dirty	 	 4
 #define PG_lru			 5
-#define PG_active		 6
+#define PG_reclaim1		 6	/* reserved by the mm reclaim code */
 #define PG_slab			 7	/* slab debug (Suparna wants this) */
 
 #define PG_checked		 8	/* kill me in 2.5.<early>. */

@@ -259,11 +259,6 @@ extern void __mod_page_state_offset(unsi
 #define ClearPageLRU(page)	clear_bit(PG_lru, &(page)->flags)
 #define __ClearPageLRU(page)	__clear_bit(PG_lru, &(page)->flags)
 
-#define PageActive(page)	test_bit(PG_active, &(page)->flags)
-#define SetPageActive(page)	set_bit(PG_active, &(page)->flags)
-#define ClearPageActive(page)	clear_bit(PG_active, &(page)->flags)
-#define __ClearPageActive(page)	__clear_bit(PG_active, &(page)->flags)
-
 #define PageSlab(page)		test_bit(PG_slab, &(page)->flags)
 #define __SetPageSlab(page)	__set_bit(PG_slab, &(page)->flags)
 #define __ClearPageSlab(page)	__clear_bit(PG_slab, &(page)->flags)

Index: linux-2.6/mm/page_alloc.c
===================================================================
@@ -149,7 +149,7 @@ static void bad_page(struct page *page)
 	page->flags &= ~(1 << PG_lru	|
 			1 << PG_private |
 			1 << PG_locked	|
-			1 << PG_active	|
+			1 << PG_reclaim1 |
 			1 << PG_dirty	|
 			1 << PG_reclaim |
 			1 << PG_slab    |

@@ -379,7 +379,7 @@ static inline int free_pages_check(struc
 			1 << PG_lru	|
 			1 << PG_private |
 			1 << PG_locked	|
-			1 << PG_active	|
+			1 << PG_reclaim1 |
 			1 << PG_reclaim	|
 			1 << PG_slab	|
 			1 << PG_swapcache |

@@ -527,7 +527,7 @@ static int prep_new_page(struct page *pa
 			1 << PG_lru	|
 			1 << PG_private	|
 			1 << PG_locked	|
-			1 << PG_active	|
+			1 << PG_reclaim1 |
 			1 << PG_dirty	|
 			1 << PG_reclaim	|
 			1 << PG_slab    |

Index: linux-2.6/mm/hugetlb.c
===================================================================
@@ -291,7 +291,7 @@ static void update_and_free_page(struct 
 	nr_huge_pages_node[page_zone(page)->zone_pgdat->node_id]--;
 	for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
 		page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
-				1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
+				1 << PG_dirty | 1 << PG_reclaim1 | 1 << PG_reserved |
 				1 << PG_private | 1<< PG_writeback);
 	}
 	page[1].lru.next = NULL;

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -165,6 +165,9 @@ static inline void __pgrep_remove(struct
 		zone->policy.nr_inactive--;
 }
 
+#define pgrep_remember(z, p) do { } while (0)
+#define pgrep_forget(m, i) do { } while (0)
+
 static inline unsigned long __pgrep_nr_pages(struct zone *zone)
 {
 	return zone->policy.nr_active + zone->policy.nr_inactive;

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -88,6 +88,8 @@ extern unsigned long pgrep_shrink_zone(i
 /* int pgrep_is_active(struct page *); */
 /* void __pgrep_remove(struct zone *zone, struct page *page); */
 extern void pgrep_reinsert(struct list_head *);
+/* void pgrep_remember(struct zone *, struct page*); */
+/* void pgrep_forget(struct address_space *, unsigned long); */
 extern void pgrep_show(struct zone *);
 extern void pgrep_zoneinfo(struct zone *, struct seq_file *);
 extern void __pgrep_counts(unsigned long *, unsigned long *,

Index: linux-2.6/mm/memory.c
===================================================================
@@ -603,6 +603,31 @@ int copy_page_range(struct mm_struct *ds
 	return 0;
 }
 
+#if defined MM_POLICY_HAS_NONRESIDENT
+static void free_file(struct vm_area_struct *vma,
+				unsigned long offset)
+{
+	struct address_space *mapping;
+	struct page *page;
+
+	if (!vma ||
+	    !vma->vm_file ||
+	    !vma->vm_file->f_mapping)
+		return;
+
+	mapping = vma->vm_file->f_mapping;
+	page = find_get_page(mapping, offset);
+	if (page) {
+		page_cache_release(page);
+		return;
+	}
+
+	pgrep_forget(mapping, offset);
+}
+#else
+#define free_file(a,b) do { } while (0)
+#endif
+
 static unsigned long zap_pte_range(struct mmu_gather *tlb,
 				struct vm_area_struct *vma, pmd_t *pmd,
 				unsigned long addr, unsigned long end,

@@ -618,6 +643,7 @@ static unsigned long zap_pte_range(struc
 	do {
 		pte_t ptent = *pte;
 		if (pte_none(ptent)) {
+			free_file(vma, pte_to_pgoff(ptent));
 			(*zap_work)--;
 			continue;
 		}

@@ -677,6 +703,8 @@ static unsigned long zap_pte_range(struc
 			continue;
 		if (!pte_file(ptent))
 			free_swap_and_cache(pte_to_swp_entry(ptent));
+		else
+			free_file(vma, pte_to_pgoff(ptent));
 		pte_clear_full(mm, addr, pte, tlb->fullmm);
 	} while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0));
 

Index: linux-2.6/mm/swapfile.c
===================================================================
@@ -28,6 +28,7 @@
 #include <linux/mutex.h>
 #include <linux/capability.h>
 #include <linux/syscalls.h>
+#include <linux/mm_page_replace.h>
 
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>

@@ -300,7 +301,8 @@ void swap_free(swp_entry_t entry)
 
 	p = swap_info_get(entry);
 	if (p) {
-		swap_entry_free(p, swp_offset(entry));
+		if (!swap_entry_free(p, swp_offset(entry)))
+			pgrep_forget(&swapper_space, entry.val);
 		spin_unlock(&swap_lock);
 	}
 }

@@ -397,12 +399,18 @@ void free_swap_and_cache(swp_entry_t ent
 
 	p = swap_info_get(entry);
 	if (p) {
-		if (swap_entry_free(p, swp_offset(entry)) == 1) {
+		switch (swap_entry_free(p, swp_offset(entry))) {
+		case 1:
 			page = find_get_page(&swapper_space, entry.val);
 			if (page && unlikely(TestSetPageLocked(page))) {
 				page_cache_release(page);
 				page = NULL;
 			}
+			break;
+
+		case 0:
+			pgrep_forget(&swapper_space, entry.val);
+			break;
 		}
 		spin_unlock(&swap_lock);
 	}

Index: linux-2.6/mm/vmscan.c
===================================================================
@@ -308,6 +308,7 @@ int remove_mapping(struct address_space 
 
 	if (PageSwapCache(page)) {
 		swp_entry_t swap = { .val = page_private(page) };
+		pgrep_remember(page_zone(page), page);
 		__delete_from_swap_cache(page);
 		write_unlock_irq(&mapping->tree_lock);
 		swap_free(swap);

@@ -315,6 +316,7 @@ int remove_mapping(struct address_space 
 		return 1;
 	}
 
+	pgrep_remember(page_zone(page), page);
 	__remove_from_page_cache(page);
 	write_unlock_irq(&mapping->tree_lock);
 	__put_page(page);

Index: linux-2.6/include/linux/mm_page_replace.h
===================================================================
@@ -114,5 +114,12 @@ static inline void pgrep_add_drain(void)
 	put_cpu();
 }
 
+#if ! defined MM_POLICY_HAS_SHRINKER
+/* unsigned long __pgrep_nr_scan(struct zone *); */
+void __pgrep_get_candidates(struct zone *, int, unsigned long, struct list_head *,
+		unsigned long *);
+void pgrep_put_candidates(struct zone *, struct list_head *, unsigned long, int);
+#endif
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_PAGE_REPLACE_H */

66 lines not on this page · raw message

Index: linux-2.6/include/linux/mm_use_once_policy.h
===================================================================
@@ -173,5 +173,7 @@ static inline unsigned long __pgrep_nr_p
 	return zone->policy.nr_active + zone->policy.nr_inactive;
 }
 
+#define MM_POLICY_HAS_SHRINKER
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_USEONCE_POLICY_H */

Index: linux-2.6/include/linux/percpu.h
===================================================================
@@ -15,6 +15,11 @@
 #define get_cpu_var(var) (*({ preempt_disable(); &__get_cpu_var(var); }))
 #define put_cpu_var(var) preempt_enable()
 
+#define __sum_cpu_var(type, var) ({ __typeof__(type) sum = 0; \
+                                 int cpu; \
+                                 for_each_cpu(cpu) sum += per_cpu(var, cpu); \
+                                 sum; })
+
 #ifdef CONFIG_SMP
 
 struct percpu_data {

Index: linux-2.6/mm/nonresident.c
===================================================================
@@ -0,0 +1,175 @@
+/*
+ * mm/nonresident.c
+ * (C) 2004,2005 Red Hat, Inc
+ * Written by Rik van Riel <[email protected]>
+ * Released under the GPL, see the file COPYING for details.
+ *
+ * Keeps track of whether a non-resident page was recently evicted
+ * and should be immediately promoted to the active list. This also
+ * helps automatically tune the inactive target.
+ *
+ * The pageout code stores a recently evicted page in this cache
+ * by calling remember_page(mapping/mm, index/vaddr, generation)
+ * and can look it up in the cache by calling recently_evicted()
+ * with the same arguments.
+ *
+ * Note that there is no way to invalidate pages after eg. truncate
+ * or exit, we let the pages fall out of the non-resident set through
+ * normal replacement.
+ */
+#include <linux/mm.h>
+#include <linux/cache.h>
+#include <linux/spinlock.h>
+#include <linux/bootmem.h>
+#include <linux/hash.h>
+#include <linux/prefetch.h>
+#include <linux/kernel.h>
+
+/* Number of non-resident pages per hash bucket. Never smaller than 15. */
+#if (L1_CACHE_BYTES < 64)
+#define NR_BUCKET_BYTES 64
+#else
+#define NR_BUCKET_BYTES L1_CACHE_BYTES
+#endif
+#define NUM_NR ((NR_BUCKET_BYTES - sizeof(atomic_t))/sizeof(u32))
+
+struct nr_bucket
+{
+	atomic_t hand;
+	u32 page[NUM_NR];
+} ____cacheline_aligned;
+
+/* The non-resident page hash table. */
+static struct nr_bucket * nonres_table;
+static unsigned int nonres_shift;
+static unsigned int nonres_mask;
+
+static struct nr_bucket * nr_hash(void * mapping, unsigned long index)
+{
+	unsigned long bucket;
+	unsigned long hash;
+
+	hash = hash_ptr(mapping, BITS_PER_LONG);
+	hash = 37 * hash + hash_long(index, BITS_PER_LONG);
+	bucket = hash & nonres_mask;
+
+	return nonres_table + bucket;
+}
+
+static u32 nr_cookie(struct address_space * mapping, unsigned long index)
+{
+	/*
+	 * Different hash magic from bucket selection to insure
+	 * the combined bits extend hash-space.
+	 */
+	unsigned long cookie = hash_long(index, BITS_PER_LONG);
+	cookie = 51 * cookie + hash_ptr(mapping, BITS_PER_LONG);
+
+	if (mapping && mapping->host) {
+		cookie = 37 * cookie + hash_long(mapping->host->i_ino, BITS_PER_LONG);
+	}
+
+	return (u32)(cookie >> (BITS_PER_LONG - 32));
+}
+
+unsigned long nonresident_get(struct address_space * mapping, unsigned long index)
+{
+	struct nr_bucket * nr_bucket;
+	int distance;
+	u32 wanted;
+	int i;
+
+	prefetch(mapping->host);
+	nr_bucket = nr_hash(mapping, index);
+
+	prefetch(nr_bucket);
+	wanted = nr_cookie(mapping, index);
+
+	for (i = 0; i < NUM_NR; i++) {
+		if (nr_bucket->page[i] == wanted) {
+			nr_bucket->page[i] = 0;
+			/* Return the distance between entry and clock hand. */
+			distance = atomic_read(&nr_bucket->hand) + NUM_NR - i;
+			distance = (distance % NUM_NR) << nonres_shift;
+			/*
+			 * Add some jitter to the lower nonres_shift bits.
+			 */
+			distance += (nr_bucket - nonres_table);
+			return distance;
+		}
+	}
+
+	return ~0UL;
+}
+
+u32 nonresident_put(struct address_space * mapping, unsigned long index)
+{
+	struct nr_bucket * nr_bucket;
+	u32 nrpage;
+	int i;
+
+	prefetch(mapping->host);
+	nr_bucket = nr_hash(mapping, index);
+
+	prefetchw(nr_bucket);
+	nrpage = nr_cookie(mapping, index);
+
+	/* Atomically find the next array index. */
+	preempt_disable();
+retry:
+	i = atomic_inc_return(&nr_bucket->hand);
+	if (unlikely(i >= NUM_NR)) {
+		if (i == NUM_NR)
+			atomic_set(&nr_bucket->hand, -1);
+		goto retry;
+	}
+	preempt_enable();
+
+	/* Statistics may want to know whether the entry was in use. */
+	return xchg(&nr_bucket->page[i], nrpage);
+}
+
+unsigned long fastcall nonresident_total(void)
+{
+	return NUM_NR << nonres_shift;
+}
+
+/*
+ * For interactive workloads, we remember about as many non-resident pages
+ * as we have actual memory pages.  For server workloads with large inter-
+ * reference distances we could benefit from remembering more.
+ */
+static __initdata unsigned long nonresident_factor = 1;
+void __init nonresident_init(void)
+{
+	int target;
+	int i;
+
+	/*
+	 * Calculate the non-resident hash bucket target. Use a power of
+	 * two for the division because alloc_large_system_hash rounds up.
+	 */
+	target = nr_all_pages * nonresident_factor;
+	target /= (sizeof(struct nr_bucket) / sizeof(u32));
+
+	nonres_table = alloc_large_system_hash("Non-resident page tracking",
+					sizeof(struct nr_bucket),
+					target,
+					0,
+					HASH_EARLY | HASH_HIGHMEM,
+					&nonres_shift,
+					&nonres_mask,
+					0);
+
+	for (i = 0; i < (1 << nonres_shift); i++)
+		atomic_set(&nonres_table[i].hand, 0);
+}
+
+static int __init set_nonresident_factor(char * str)
+{
+	if (!str)
+		return 0;
+	nonresident_factor = simple_strtoul(str, &str, 0);
+	return 1;
+}
+__setup("nonresident_factor=", set_nonresident_factor);

Index: linux-2.6/include/linux/nonresident.h
===================================================================
@@ -0,0 +1,12 @@
+#ifndef _LINUX_NONRESIDENT_H_
+#define _LINUX_NONRESIDENT_H_
+
+#ifdef __KERNEL__
+
+extern void nonresident_init(void);
+extern unsigned long nonresident_get(struct address_space *, unsigned long);
+extern u32 nonresident_put(struct address_space *, unsigned long);
+extern unsigned long fastcall nonresident_total(void);
+
+#endif /* __KERNEL */
+#endif /* _LINUX_NONRESIDENT_H_ */

Index: linux-2.6/include/linux/rmap.h
===================================================================
@@ -90,7 +90,7 @@ static inline void page_dup_rmap(struct 
 /*
  * Called from mm/vmscan.c to handle paging out
  */
-int page_referenced(struct page *, int is_locked);
+int page_referenced(struct page *, int is_locked, int ignore_token);
 int try_to_unmap(struct page *, int ignore_refs);
 void remove_from_swap(struct page *page);
 

@@ -111,7 +111,7 @@ unsigned long page_address_in_vma(struct
 #define anon_vma_prepare(vma)	(0)
 #define anon_vma_link(vma)	do {} while (0)
 
-#define page_referenced(page,l) TestClearPageReferenced(page)
+#define page_referenced(page,l,i) TestClearPageReferenced(page)
 #define try_to_unmap(page, refs) SWAP_FAIL
 
 #endif	/* CONFIG_MMU */

Index: linux-2.6/mm/rmap.c
===================================================================
@@ -328,7 +328,7 @@ pte_t *page_check_address(struct page *p
  * repeatedly from either page_referenced_anon or page_referenced_file.
  */
 static int page_referenced_one(struct page *page,
-	struct vm_area_struct *vma, unsigned int *mapcount)
+	struct vm_area_struct *vma, unsigned int *mapcount, int ignore_token)
 {
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long address;

@@ -349,7 +349,7 @@ static int page_referenced_one(struct pa
 
 	/* Pretend the page is referenced if the task has the
 	   swap token and is in the middle of a page fault. */
-	if (mm != current->mm && has_swap_token(mm) &&
+	if (mm != current->mm && !ignore_token && has_swap_token(mm) &&
 			rwsem_is_locked(&mm->mmap_sem))
 		referenced++;
 

@@ -359,7 +359,7 @@ out:
 	return referenced;
 }
 
-static int page_referenced_anon(struct page *page)
+static int page_referenced_anon(struct page *page, int ignore_token)
 {
 	unsigned int mapcount;
 	struct anon_vma *anon_vma;

@@ -372,7 +372,8 @@ static int page_referenced_anon(struct p
 
 	mapcount = page_mapcount(page);
 	list_for_each_entry(vma, &anon_vma->head, anon_vma_node) {
-		referenced += page_referenced_one(page, vma, &mapcount);
+		referenced += page_referenced_one(page, vma, &mapcount,
+							ignore_token);
 		if (!mapcount)
 			break;
 	}

@@ -391,7 +392,7 @@ static int page_referenced_anon(struct p
  *
  * This function is only called from page_referenced for object-based pages.
  */
-static int page_referenced_file(struct page *page)
+static int page_referenced_file(struct page *page, int ignore_token)
 {
 	unsigned int mapcount;
 	struct address_space *mapping = page->mapping;

@@ -429,7 +430,8 @@ static int page_referenced_file(struct p
 			referenced++;
 			break;
 		}
-		referenced += page_referenced_one(page, vma, &mapcount);
+		referenced += page_referenced_one(page, vma, &mapcount,
+							ignore_token);
 		if (!mapcount)
 			break;
 	}