O(1) vm (rmap15) for 2.4.20-rc2-ac1

diff -Nru a/fs/proc/proc_misc.c b/fs/proc/proc_misc.c
@@ -192,6 +192,7 @@
 		"SwapCached:   %8lu kB\n"
 		"Active:       %8u kB\n"
 		"Inact_dirty:  %8u kB\n"
+		"Inact_laundry:%8u kB\n"
 		"Inact_clean:  %8u kB\n"
 		"Inact_target: %8u kB\n"
 		"HighTotal:    %8lu kB\n"

@@ -209,6 +210,7 @@
 		K(swapper_space.nrpages),
 		K(nr_active_pages),
 		K(nr_inactive_dirty_pages),
+		K(nr_inactive_laundry_pages),
 		K(nr_inactive_clean_pages),
 		K(inactive_target()),
 		K(i.totalhigh),

diff -Nru a/include/linux/brlock.h b/include/linux/brlock.h
@@ -37,6 +37,8 @@
 	BR_GLOBALIRQ_LOCK,
 	BR_NETPROTO_LOCK,
 	BR_LLC_LOCK,
+	BR_LRU_LOCK,
+
 	__BR_END
 };
 

diff -Nru a/include/linux/list.h b/include/linux/list.h
@@ -137,8 +137,7 @@
 	return head->next == head;
 }
 
-static inline void __list_splice(struct list_head *list,
-				 struct list_head *head)
+static inline void __list_splice(struct list_head *list, struct list_head *head)
 {
 	struct list_head *first = list->next;
 	struct list_head *last = list->prev;

86 lines not on this page · raw message

179 lines not on this page · raw message

diff -Nru a/include/linux/mmzone.h b/include/linux/mmzone.h
@@ -13,11 +13,7 @@
  * Free memory management - zoned buddy allocator.
  */
 
-#ifndef CONFIG_FORCE_MAX_ZONEORDER
 #define MAX_ORDER 10
-#else
-#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
-#endif
 
 typedef struct free_area_struct {
 	struct list_head	free_list;

@@ -29,6 +25,9 @@
 
 #define MAX_CHUNKS_PER_NODE 8
 
+#define MAX_AGE 15
+#define INITIAL_AGE 3
+
 #define MAX_PER_CPU_PAGES 512
 typedef struct per_cpu_pages_s {
 	int			nr_pages, max_nr_pages;

@@ -52,17 +51,22 @@
 	unsigned long		free_pages;
 	unsigned long		active_pages;
 	unsigned long		inactive_dirty_pages;
+	unsigned long		inactive_laundry_pages;
 	unsigned long		inactive_clean_pages;
 	unsigned long		pages_min, pages_low, pages_high, pages_plenty;
 	int			need_balance;
+	int			need_scan;
+	int			active_count[MAX_AGE+1];
 
 	/*
 	 * free areas of different sizes
 	 */
-	struct list_head	active_list;
+	struct list_head	active_list[MAX_AGE+1];
 	struct list_head	inactive_dirty_list;
+	struct list_head	inactive_laundry_list;
 	struct list_head	inactive_clean_list;
 	free_area_t		free_area[MAX_ORDER];
+	spinlock_t		lru_lock;
 
 	/*
 	 * wait_table           -- the array holding the hash table

diff -Nru a/include/linux/module.h b/include/linux/module.h
@@ -287,6 +287,9 @@
 static const char __module_license[] __attribute__((section(".modinfo"))) =   \
 "license=" license
 
+#define GPL_HEADER() \
+static const char cpyright="This software may be freely redistributed under the terms of the GNU General Public License.";
+
 /* Define the module variable, and usage macros.  */
 extern struct module __this_module;
 

@@ -302,7 +305,6 @@
 static const char __module_using_checksums[] __attribute__((section(".modinfo"))) =
 "using_checksums=1";
 #endif
-
 #else /* MODULE */
 
 #define MODULE_AUTHOR(name)

@@ -311,6 +313,7 @@
 #define MODULE_SUPPORTED_DEVICE(name)
 #define MODULE_PARM(var,type)
 #define MODULE_PARM_DESC(var,desc)
+#define GPL_HEADER()
 
 /* Create a dummy reference to the table to suppress gcc unused warnings.  Put
  * the reference in the .data.exit section which is discarded when code is built

diff -Nru a/include/linux/pagemap.h b/include/linux/pagemap.h
@@ -70,10 +70,6 @@
 
 #define page_hash(mapping,index) (page_hash_table+_page_hashfn(mapping,index))
 
-extern struct page * __find_get_page(struct address_space *mapping,
-				unsigned long index, struct page **hash);
-#define find_get_page(mapping, index) \
-	__find_get_page(mapping, index, page_hash(mapping, index))
 extern struct page * __find_lock_page (struct address_space * mapping,
 				unsigned long index, struct page **hash);
 extern struct page * find_or_create_page(struct address_space *mapping,

@@ -90,6 +86,13 @@
 extern int add_to_page_cache_unique(struct page * page, struct address_space *mapping, unsigned long index, struct page **hash);
 
 extern void ___wait_on_page(struct page *);
+extern int wait_on_page_timeout(struct page *page, int timeout);
+
+
+extern struct page * __find_pagecache_page(struct address_space *mapping,
+				unsigned long index, struct page **hash);
+#define find_pagecache_page(mapping, index) \
+	__find_pagecache_page(mapping, index, page_hash(mapping, index))
 
 static inline void wait_on_page(struct page * page)
 {

diff -Nru a/include/linux/swap.h b/include/linux/swap.h
@@ -87,6 +87,7 @@
 extern unsigned int nr_free_buffer_pages(void);
 extern int nr_active_pages;
 extern int nr_inactive_dirty_pages;
+extern int nr_inactive_laundry_pages;
 extern int nr_inactive_clean_pages;
 extern atomic_t page_cache_size;
 extern atomic_t buffermem_pages;

@@ -115,6 +116,7 @@
 
 /* linux/mm/swap.c */
 extern void FASTCALL(lru_cache_add(struct page *));
+extern void FASTCALL(lru_cache_add_dirty(struct page *));
 extern void FASTCALL(__lru_cache_del(struct page *));
 extern void FASTCALL(lru_cache_del(struct page *));
 

@@ -175,8 +177,6 @@
 asmlinkage long sys_swapoff(const char *);
 asmlinkage long sys_swapon(const char *, int);
 
-extern spinlock_cacheline_t pagemap_lru_lock_cacheline;
-#define pagemap_lru_lock pagemap_lru_lock_cacheline.lock
 
 extern void FASTCALL(mark_page_accessed(struct page *));
 

@@ -191,7 +191,7 @@
 
 /*
  * List add/del helper macros. These must be called
- * with the pagemap_lru_lock held!
+ * with the lru lock held!
  */
 #define DEBUG_LRU_PAGE(page)			\
 do {						\

diff -Nru a/kernel/ksyms.c b/kernel/ksyms.c
@@ -262,7 +262,6 @@
 EXPORT_SYMBOL(__pollwait);
 EXPORT_SYMBOL(poll_freewait);
 EXPORT_SYMBOL(ROOT_DEV);
-EXPORT_SYMBOL(__find_get_page);
 EXPORT_SYMBOL(__find_lock_page);
 EXPORT_SYMBOL(find_or_create_page);
 EXPORT_SYMBOL(grab_cache_page_nowait);

155 lines not on this page · raw message

diff -Nru a/mm/page_alloc.c b/mm/page_alloc.c
@@ -28,6 +28,7 @@
 int nr_swap_pages;
 int nr_active_pages;
 int nr_inactive_dirty_pages;
+int nr_inactive_laundry_pages;
 int nr_inactive_clean_pages;
 pg_data_t *pgdat_list;
 

@@ -113,12 +114,13 @@
 		BUG();
 	if (PageInactiveDirty(page))
 		BUG();
+	if (PageInactiveLaundry(page))
+		BUG();
 	if (PageInactiveClean(page))
 		BUG();
 	if (page->pte_chain)
 		BUG();
 	page->flags &= ~((1<<PG_referenced) | (1<<PG_dirty));
-	page->age = PAGE_AGE_START;
 	
 	zone = page_zone(page);
 

@@ -728,9 +730,10 @@
 		nr_free_pages() << (PAGE_SHIFT-10),
 		nr_free_highpages() << (PAGE_SHIFT-10));
 
-	printk("( Active: %d, inactive_dirty: %d, inactive_clean: %d, free: %d )\n",
+	printk("( Active: %d/%d, inactive_laundry: %d, inactive_clean: %d, free: %d )\n",
 		nr_active_pages,
 		nr_inactive_dirty_pages,
+		nr_inactive_laundry_pages,
 		nr_inactive_clean_pages,
 		nr_free_pages());
 

@@ -941,12 +944,20 @@
 		zone->lock = SPIN_LOCK_UNLOCKED;
 		zone->zone_pgdat = pgdat;
 		zone->free_pages = 0;
+		zone->active_pages = 0;
 		zone->inactive_clean_pages = 0;
+		zone->inactive_laundry_pages = 0;
 		zone->inactive_dirty_pages = 0;
 		zone->need_balance = 0;
-		INIT_LIST_HEAD(&zone->active_list);
+		zone->need_scan = 0;
+		for (k = 0; k <= MAX_AGE ; k++) {
+			INIT_LIST_HEAD(&zone->active_list[k]);
+			zone->active_count[k] = 0;
+		}
 		INIT_LIST_HEAD(&zone->inactive_dirty_list);
+		INIT_LIST_HEAD(&zone->inactive_laundry_list);
 		INIT_LIST_HEAD(&zone->inactive_clean_list);
+		spin_lock_init(&zone->lru_lock);
 
 		if (!size)
 			continue;

diff -Nru a/mm/rmap.c b/mm/rmap.c
@@ -14,7 +14,7 @@
 /*
  * Locking:
  * - the page->pte_chain is protected by the PG_chainlock bit,
- *   which nests within the pagemap_lru_lock, then the
+ *   which nests within the lru lock, then the
  *   mm->page_table_lock, and then the page lock.
  * - because swapout locking is opposite to the locking order
  *   in the page fault path, the swapout path uses trylocks

@@ -195,7 +195,7 @@
  * table entry mapping a page. Because locking order here is opposite
  * to the locking order used by the page fault path, we use trylocks.
  * Locking:
- *	pagemap_lru_lock		page_launder()
+ *	   lru lock			page_launder()
  *	    page lock			page_launder(), trylock
  *		pte_chain_lock		page_launder()
  *		    mm->page_table_lock	try_to_unmap_one(), trylock

@@ -263,7 +263,7 @@
  * @page: the page to get unmapped
  *
  * Tries to remove all the page table entries which are mapping this
- * page, used in the pageout path.  Caller must hold pagemap_lru_lock
+ * page, used in the pageout path.  Caller must hold lru lock
  * and the page lock.  Return values are:
  *
  * SWAP_SUCCESS	- we succeeded in removing all mappings

diff -Nru a/mm/shmem.c b/mm/shmem.c
@@ -581,7 +581,7 @@
 	 * cache and swap cache.  We need to recheck the page cache
 	 * under the protection of the info->lock spinlock. */
 
-	page = find_get_page(mapping, idx);
+	page = find_pagecache_page(mapping, idx);
 	if (page) {
 		if (TryLockPage(page))
 			goto wait_retry;

164 lines not on this page · raw message

diff -Nru a/mm/swap_state.c b/mm/swap_state.c
@@ -196,7 +196,7 @@
 {
 	struct page *found;
 
-	found = find_get_page(&swapper_space, entry.val);
+	found = find_pagecache_page(&swapper_space, entry.val);
 	/*
 	 * Unsafe to assert PageSwapCache and mapping on page found:
 	 * if SMP nothing prevents swapoff from deleting this page from

@@ -224,10 +224,10 @@
 		/*
 		 * First check the swap cache.  Since this is normally
 		 * called after lookup_swap_cache() failed, re-calling
-		 * that would confuse statistics: use find_get_page()
+		 * that would confuse statistics: use find_pagecache_page()
 		 * directly.
 		 */
-		found_page = find_get_page(&swapper_space, entry.val);
+		found_page = find_pagecache_page(&swapper_space, entry.val);
 		if (found_page)
 			break;
 

1,141 lines not on this page · raw message