hashed spinlocking for pte_chain manipulation

@@ -15,6 +15,7 @@
  *   offset of the page table entry within the page table page
  */
 #include <linux/mm.h>
+#include <linux/rmap-locking.h>
 
 static inline void pgtable_add_rmap(struct page * page, struct mm_struct * mm, unsigned long address)
 {

@@ -65,9 +65,7 @@
 #define PG_private		12	/* Has something at ->private */
 #define PG_writeback		13	/* Page is under writeback */
 #define PG_nosave		14	/* Used for system suspend/resume */
-#define PG_chainlock		15	/* lock bit for ->pte_chain */
-
-#define PG_direct		16	/* ->pte_chain points directly at pte */
+#define PG_direct		15	/* ->pte_chain points directly at pte */
 
 /*
  * Global page accounting.  One instance per CPU.

@@ -230,36 +228,6 @@ extern void get_page_state(struct page_s
 #define TestClearPageDirect(page)	test_and_clear_bit(PG_direct, &(page)->flags)
 
 /*
- * inlines for acquisition and release of PG_chainlock
- */
-static inline void pte_chain_lock(struct page *page)
-{
-	/*
-	 * Assuming the lock is uncontended, this never enters
-	 * the body of the outer loop. If it is contended, then
-	 * within the inner loop a non-atomic test is used to
-	 * busywait with less bus contention for a good time to
-	 * attempt to acquire the lock bit.
-	 */
-	preempt_disable();
-#ifdef CONFIG_SMP
-	while (test_and_set_bit(PG_chainlock, &page->flags)) {
-		while (test_bit(PG_chainlock, &page->flags))
-			cpu_relax();
-	}
-#endif
-}
-
-static inline void pte_chain_unlock(struct page *page)
-{
-#ifdef CONFIG_SMP
-	smp_mb__before_clear_bit();
-	clear_bit(PG_chainlock, &page->flags);
-#endif
-	preempt_enable();
-}
-
-/*
  * The PageSwapCache predicate doesn't use a PG_flag at this time,
  * but it may again do so one day.
  */

109 lines not on this page · raw message

@@ -144,7 +144,9 @@ struct pagevec;
 
 /* linux/mm/rmap.c */
 extern int FASTCALL(page_referenced(struct page *));
+extern void FASTCALL(__page_add_rmap(struct page *, pte_t *));
 extern void FASTCALL(page_add_rmap(struct page *, pte_t *));
+extern void FASTCALL(__page_remove_rmap(struct page *, pte_t *));
 extern void FASTCALL(page_remove_rmap(struct page *, pte_t *));
 extern int FASTCALL(try_to_unmap(struct page *));
 extern int FASTCALL(page_over_rsslimit(struct page *));

@@ -53,13 +53,20 @@
 /*
  * Lock ordering:
  *
- *  ->i_shared_lock		(vmtruncate)
- *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
+ *  ->i_shared_lock			(vmtruncate)
+ *    ->private_lock			(__free_pte->__set_page_dirty_buffers)
  *      ->swap_list_lock
- *        ->swap_device_lock	(exclusive_swap_page, others)
- *          ->mapping->page_lock
- *      ->inode_lock		(__mark_inode_dirty)
- *        ->sb_lock		(fs/fs-writeback.c)
+ *        ->swap_device_lock		(exclusive_swap_page, others)
+ *	    ->rmap_lock			(to/from swapcache)
+ *            ->mapping->page_lock
+ *		->pagemap_lru_lock	(zap_pte_range)
+ *      ->inode_lock			(__mark_inode_dirty)
+ *        ->sb_lock			(fs/fs-writeback.c)
+ *
+ *  mm->page_table_lock
+ *    ->rmap_lock			(copy_page_range)
+ *    ->mapping->page_lock		(try_to_unmap_one)
+ *
  */
 spinlock_t _pagemap_lru_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 

170 lines not on this page · raw message

147 lines not on this page · raw message

@@ -16,6 +16,7 @@
 #include <linux/buffer_head.h>	/* block_sync_page() */
 #include <linux/pagevec.h>
 
+#include <asm/rmap.h>
 #include <asm/pgtable.h>
 
 /*

@@ -71,6 +72,12 @@ int add_to_swap_cache(struct page *page,
 		INC_CACHE_INFO(noent_race);
 		return -ENOENT;
 	}
+
+	/*
+	 * Sneakily do this here so we don't add cost to add_to_page_cache().
+	 */
+	set_page_index(page, entry.val);
+
 	error = add_to_page_cache(page, &swapper_space, entry.val);
 	/*
 	 * Anon pages are already on the LRU, we don't run lru_cache_add here.

@@ -204,6 +211,7 @@ int move_to_swap_cache(struct page *page
 		return -ENOENT;
 	}
 
+	set_page_index(page, entry.val);
 	write_lock(&swapper_space.page_lock);
 	write_lock(&mapping->page_lock);
 

@@ -220,7 +228,6 @@ int move_to_swap_cache(struct page *page
 		 */
 		ClearPageUptodate(page);
 		ClearPageReferenced(page);
-
 		SetPageLocked(page);
 		ClearPageDirty(page);
 		___add_to_page_cache(page, &swapper_space, entry.val);

@@ -252,6 +259,7 @@ int move_from_swap_cache(struct page *pa
 	BUG_ON(PageWriteback(page));
 	BUG_ON(page_has_buffers(page));
 
+	set_page_index(page, index);
 	write_lock(&swapper_space.page_lock);
 	write_lock(&mapping->page_lock);
 

@@ -25,6 +25,7 @@
 #include <linux/buffer_head.h>		/* for try_to_release_page() */
 #include <linux/pagevec.h>
 
+#include <asm/rmap.h>
 #include <asm/pgalloc.h>
 #include <asm/tlbflush.h>
 #include <linux/swapops.h>

@@ -67,7 +68,7 @@
 #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
 #endif
 
-/* Must be called with page's pte_chain_lock held. */
+/* Must be called with page's rmap_lock held. */
 static inline int page_mapping_inuse(struct page * page)
 {
 	struct address_space *mapping = page->mapping;

@@ -106,6 +107,7 @@ shrink_list(struct list_head *page_list,
 	while (!list_empty(page_list)) {
 		struct page *page;
 		int may_enter_fs;
+		spinlock_t *rmap_lock;
 
 		page = list_entry(page_list->prev, struct page, lru);
 		list_del(&page->lru);

@@ -125,10 +127,10 @@ shrink_list(struct list_head *page_list,
 				goto keep_locked;
 		}
 
-		pte_chain_lock(page);
+		rmap_lock = lock_rmap(page);
 		if (page_referenced(page) && page_mapping_inuse(page)) {
 			/* In active use or really unfreeable.  Activate it. */
-			pte_chain_unlock(page);
+			unlock_rmap(rmap_lock);
 			goto activate_locked;
 		}
 

@@ -139,10 +141,10 @@ shrink_list(struct list_head *page_list,
 		 * XXX: implement swap clustering ?
 		 */
 		if (page->pte.chain && !page->mapping && !PagePrivate(page)) {
-			pte_chain_unlock(page);
+			unlock_rmap(rmap_lock);
 			if (!add_to_swap(page))
 				goto activate_locked;
-			pte_chain_lock(page);
+			rmap_lock = lock_rmap(page);
 		}
 
 		/*

@@ -153,16 +155,16 @@ shrink_list(struct list_head *page_list,
 			switch (try_to_unmap(page)) {
 			case SWAP_ERROR:
 			case SWAP_FAIL:
-				pte_chain_unlock(page);
+				unlock_rmap(rmap_lock);
 				goto activate_locked;
 			case SWAP_AGAIN:
-				pte_chain_unlock(page);
+				unlock_rmap(rmap_lock);
 				goto keep_locked;
 			case SWAP_SUCCESS:
 				; /* try to free the page below */
 			}
 		}
-		pte_chain_unlock(page);
+		unlock_rmap(rmap_lock);
 		mapping = page->mapping;
 
 		/*

@@ -380,6 +382,8 @@ static /* inline */ void refill_inactive
 	LIST_HEAD(l_active);	/* Pages to go onto the active_list */
 	struct page *page;
 	struct pagevec pvec;
+	spinlock_t *rmap_lock = NULL;
+	unsigned last_lockno = -1;
 
 	lru_add_drain();
 	spin_lock_irq(&_pagemap_lru_lock);

@@ -398,20 +402,16 @@ static /* inline */ void refill_inactive
 		page = list_entry(l_hold.prev, struct page, lru);
 		list_del(&page->lru);
 		if (page->pte.chain) {
-			if (test_and_set_bit(PG_chainlock, &page->flags)) {
-				list_add(&page->lru, &l_active);
-				continue;
-			}
+			cached_rmap_lock(page, &rmap_lock, &last_lockno);
 			if (page->pte.chain && page_referenced(page)) {
-				pte_chain_unlock(page);
 				list_add(&page->lru, &l_active);
 				continue;
 			}
-			pte_chain_unlock(page);
 		}
 		list_add(&page->lru, &l_inactive);
 		pgdeactivate++;
 	}
+	drop_rmap_lock(&rmap_lock, &last_lockno);
 
 	pagevec_init(&pvec);
 	spin_lock_irq(&_pagemap_lru_lock);