batched freeing of anonymous pages

@@ -14,6 +14,7 @@
 #include <linux/pagemap.h>
 #include <linux/smp_lock.h>
 #include <linux/buffer_head.h>	/* block_sync_page() */
+#include <linux/pagevec.h>
 
 #include <asm/pgtable.h>
 

@@ -295,7 +296,7 @@ int move_from_swap_cache(struct page *pa
  * this page if it is the last user of the page. Can not do a lock_page,
  * as we are holding the page_table_lock spinlock.
  */
-void free_page_and_swap_cache(struct page *page)
+void free_page_and_swap_cache(struct pagevec *pvec, struct page *page)
 {
 	/* 
 	 * If we are the only user, then try to free up the swap cache. 

@@ -309,7 +310,8 @@ void free_page_and_swap_cache(struct pag
 		remove_exclusive_swap_page(page);
 		unlock_page(page);
 	}
-	page_cache_release(page);
+	if (!pagevec_add(pvec, page))
+		__pagevec_release(pvec);
 }
 
 /*

@@ -14,6 +14,7 @@
 #define _ASM_GENERIC__TLB_H
 
 #include <linux/config.h>
+#include <linux/pagevec.h>
 #include <asm/tlbflush.h>
 
 /*

@@ -70,9 +71,13 @@ static inline void tlb_flush_mmu(mmu_gat
 	nr = tlb->nr;
 	if (!tlb_fast_mode(tlb)) {
 		unsigned long i;
+		struct pagevec pvec;
+
 		tlb->nr = 0;
+		pagevec_init(&pvec);
 		for (i=0; i < nr; i++)
-			free_page_and_swap_cache(tlb->pages[i]);
+			free_page_and_swap_cache(&pvec, tlb->pages[i]);
+		pagevec_release(&pvec);
 	}
 }
 

@@ -101,10 +106,11 @@ static inline void tlb_finish_mmu(mmu_ga
  *	handling the additional races in SMP caused by other CPUs caching valid
  *	mappings in their TLBs.
  */
-static inline void tlb_remove_page(mmu_gather_t *tlb, struct page *page)
+static inline void
+tlb_remove_page(mmu_gather_t *tlb, struct pagevec *pvec, struct page *page)
 {
 	if (tlb_fast_mode(tlb)) {
-		free_page_and_swap_cache(page);
+		free_page_and_swap_cache(pvec, page);
 		return;
 	}
 	tlb->pages[tlb->nr++] = page;

@@ -140,6 +140,7 @@ struct vm_area_struct;
 struct sysinfo;
 struct address_space;
 struct zone_t;
+struct pagevec;
 
 /* linux/mm/rmap.c */
 extern int FASTCALL(page_referenced(struct page *));

@@ -183,7 +184,7 @@ extern void delete_from_swap_cache(struc
 extern int move_to_swap_cache(struct page *page, swp_entry_t entry);
 extern int move_from_swap_cache(struct page *page, unsigned long index,
 		struct address_space *mapping);
-extern void free_page_and_swap_cache(struct page *page);
+extern void free_page_and_swap_cache(struct pagevec *pvec, struct page *page);
 extern struct page * lookup_swap_cache(swp_entry_t);
 extern struct page * read_swap_cache_async(swp_entry_t);
 

@@ -44,6 +44,7 @@
 #include <linux/iobuf.h>
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
+#include <linux/pagevec.h>
 
 #include <asm/pgalloc.h>
 #include <asm/rmap.h>

@@ -78,7 +79,8 @@ struct page *mem_map;
  * Note: this doesn't free the actual pages themselves. That
  * has been handled earlier when unmapping all the memory regions.
  */
-static inline void free_one_pmd(mmu_gather_t *tlb, pmd_t * dir)
+static inline void
+free_one_pmd(mmu_gather_t *tlb, struct pagevec *pvec, pmd_t * dir)
 {
 	struct page *page;
 

@@ -92,10 +94,11 @@ static inline void free_one_pmd(mmu_gath
 	page = pmd_page(*dir);
 	pmd_clear(dir);
 	pgtable_remove_rmap(page);
-	pte_free_tlb(tlb, page);
+	pte_free_tlb(tlb, pvec, page);
 }
 
-static inline void free_one_pgd(mmu_gather_t *tlb, pgd_t * dir)
+static inline void
+free_one_pgd(mmu_gather_t *tlb, struct pagevec *pvec, pgd_t *dir)
 {
 	int j;
 	pmd_t * pmd;

@@ -111,7 +114,7 @@ static inline void free_one_pgd(mmu_gath
 	pgd_clear(dir);
 	for (j = 0; j < PTRS_PER_PMD ; j++) {
 		prefetchw(pmd+j+(PREFETCH_STRIDE/16));
-		free_one_pmd(tlb, pmd+j);
+		free_one_pmd(tlb, pvec, pmd+j);
 	}
 	pmd_free_tlb(tlb, pmd);
 }

@@ -125,12 +128,15 @@ static inline void free_one_pgd(mmu_gath
 void clear_page_tables(mmu_gather_t *tlb, unsigned long first, int nr)
 {
 	pgd_t * page_dir = tlb->mm->pgd;
+	struct pagevec pvec;
 
+	pagevec_init(&pvec);
 	page_dir += first;
 	do {
-		free_one_pgd(tlb, page_dir);
+		free_one_pgd(tlb, &pvec, page_dir);
 		page_dir++;
 	} while (--nr);
+	pagevec_release(&pvec);
 }
 
 pte_t * pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address)

@@ -322,6 +328,7 @@ static void zap_pte_range(mmu_gather_t *
 {
 	unsigned long offset;
 	pte_t *ptep;
+	struct pagevec pvec;
 
 	if (pmd_none(*pmd))
 		return;

@@ -335,6 +342,7 @@ static void zap_pte_range(mmu_gather_t *
 	if (offset + size > PMD_SIZE)
 		size = PMD_SIZE - offset;
 	size &= PAGE_MASK;
+	pagevec_init(&pvec);
 	for (offset=0; offset < size; ptep++, offset += PAGE_SIZE) {
 		pte_t pte = *ptep;
 		if (pte_none(pte))

@@ -351,7 +359,7 @@ static void zap_pte_range(mmu_gather_t *
 						set_page_dirty(page);
 					tlb->freed++;
 					page_remove_rmap(page, ptep);
-					tlb_remove_page(tlb, page);
+					tlb_remove_page(tlb, &pvec, page);
 				}
 			}
 		} else {

@@ -360,6 +368,7 @@ static void zap_pte_range(mmu_gather_t *
 		}
 	}
 	pte_unmap(ptep-1);
+	pagevec_release(&pvec);
 }
 
 static void zap_pmd_range(mmu_gather_t *tlb, pgd_t * dir, unsigned long address, unsigned long size)

@@ -452,10 +452,20 @@ unsigned long get_zeroed_page(unsigned i
 
 void page_cache_release(struct page *page)
 {
+	/*
+	 * FIXME: this PageReserved test is really expensive
+	 */
 	if (!PageReserved(page) && put_page_testzero(page)) {
-		if (PageLRU(page))
-			lru_cache_del(page);
-		__free_pages_ok(page, 0);
+		/*
+		 * This path almost never happens - pages are normally freed
+		 * via pagevecs.
+		 */
+		struct pagevec pvec;
+
+		page_cache_get(page);
+		pvec.nr = 1;
+		pvec.pages[0] = page;
+		__pagevec_release(&pvec);
 	}
 }
 

@@ -16,6 +16,7 @@
 #include <linux/file.h>
 #include <linux/fs.h>
 #include <linux/personality.h>
+#include <linux/pagevec.h>
 #include <linux/security.h>
 
 #include <asm/uaccess.h>

@@ -31,9 +31,10 @@ static inline void pte_free(struct page 
         __free_page(page);
 }
 
-static inline void pte_free_tlb(mmu_gather_t *tlb, struct page *page)
+static inline void
+pte_free_tlb(mmu_gather_t *tlb, struct pagevec *pvec, struct page *page)
 {
-	tlb_remove_page(tlb, page);
+	tlb_remove_page(tlb, pvec, page);
 }
 
 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, 

@@ -37,7 +37,7 @@ static inline void pte_free(struct page 
 }
 
 
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define pte_free_tlb(tlb,pvec,pte) tlb_remove_page((tlb),(pvec),(pte))
 
 /*
  * allocating and freeing a pmd is trivial: the 1-entry pmd is

@@ -153,7 +153,7 @@ pte_free_kernel (pte_t *pte)
 	free_page((unsigned long) pte);
 }
 
-#define pte_free_tlb(tlb, pte)	tlb_remove_page((tlb), (pte))
+#define pte_free_tlb(tlb, pvec, pte)	tlb_remove_page((tlb), (pvec), (pte))
 
 extern void check_pgt_cache (void);
 

@@ -107,7 +107,7 @@ static inline void pte_free(struct page 
         __free_page(pte);
 }
 
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define pte_free_tlb(tlb,pvec,pte) tlb_remove_page((tlb),(pvec),(pte))
 
 /*
  * This establishes kernel virtual mappings (e.g., as a result of a

@@ -123,7 +123,7 @@ static inline void pte_free(struct page 
         __free_page(pte);
 }
 
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define pte_free_tlb(tlb,pvec,pte) tlb_remove_page((tlb),(pvec),(pte))
 
 /*
  * This establishes kernel virtual mappings (e.g., as a result of a

@@ -75,7 +75,7 @@ extern inline void pte_free(struct page 
 	__free_page(pte);
 } 
 
-#define pte_free_tlb(tlb,pte) tlb_remove_page((tlb),(pte))
+#define pte_free_tlb(tlb,pvec,pte) tlb_remove_page((tlb),(pvec),(pte))
 #define pmd_free_tlb(tlb,x)   do { } while (0)
 
 #endif /* _X86_64_PGALLOC_H */

@@ -16,6 +16,6 @@
 #include <asm-generic/tlb.h>
 
 #define pmd_free_tlb(tlb, pmd)	pmd_free(pmd)
-#define pte_free_tlb(tlb, pte)	pte_free(pte)
+#define pte_free_tlb(tlb, pvec, pte)	pte_free(pte)
 
 #endif

@@ -55,7 +55,8 @@ static inline void pte_free(struct page 
 	__free_page(page);
 }
 
-static inline void pte_free_tlb(mmu_gather_t *tlb, struct page *page)
+static inline void
+pte_free_tlb(mmu_gather_t *tlb, struct pagevec *pvec, struct page *page)
 {
 	cache_page(kmap(page));
 	kunmap(page);

@@ -33,7 +33,7 @@ extern struct page *pte_alloc_one(struct
 extern void pte_free_kernel(pte_t *pte);
 extern void pte_free(struct page *pte);
 
-#define pte_free_tlb(tlb, pte)	pte_free((pte))
+#define pte_free_tlb(tlb, pvec, pte)	pte_free((pte))
 
 #define check_pgt_cache()	do { } while (0)
 

@@ -22,6 +22,6 @@ do {	if (!(tlb)->fullmm)	\
 #include <asm-generic/tlb.h>
 
 #define pmd_free_tlb(tlb, pmd)	pmd_free(pmd)
-#define pte_free_tlb(tlb, pte)	pte_free(pte)
+#define pte_free_tlb(tlb, pvec, pte)	pte_free(pte)
 
 #endif /* _SPARC64_TLB_H */

@@ -87,8 +87,8 @@ pte_free_kernel(pte_t *pte)
 	free_page((unsigned long)pte);
 }
 
-#define pte_free(pte_page)	pte_free_kernel(page_address(pte_page))
-#define pte_free_tlb(tlb, pte)	pte_free(pte)
+#define pte_free(pte_page)		pte_free_kernel(page_address(pte_page))
+#define pte_free_tlb(tlb, pvec, pte)	pte_free(pte)
 
 #define check_pgt_cache()	do { } while (0)
 

@@ -1,3 +1,5 @@
+#ifndef _LINUX_PAGEVEC_H
+#define _LINUX_PAGEVEC_H
 /*
  * include/linux/pagevec.h
  *

@@ -74,3 +76,4 @@ static inline void pagevec_lru_del(struc
 	if (pagevec_count(pvec))
 		__pagevec_lru_del(pvec);
 }
+#endif _LINUX_PAGEVEC_H