per-page pte_chain locking

diff -Nru a/include/linux/mm.h b/include/linux/mm.h
@@ -299,6 +299,7 @@
 #define PG_arch_1		13
 #define PG_reserved		14
 #define PG_launder		15	/* written out by VM pressure.. */
+#define PG_chainlock		16	/* lock bit for ->pte_chain */
 
 /* Make it prettier to test the above... */
 #define UnlockPage(page)	unlock_page(page)

@@ -315,6 +316,20 @@
 #define SetPageChecked(page)	set_bit(PG_checked, &(page)->flags)
 #define PageLaunder(page)	test_bit(PG_launder, &(page)->flags)
 #define SetPageLaunder(page)	set_bit(PG_launder, &(page)->flags)
+
+/*
+ * inlines for acquisition and release of PG_chainlock
+ */
+static inline void pte_chain_lock(struct page *page)
+{
+	while (test_and_set_bit(PG_chainlock, &page->flags))
+		cpu_relax();
+}
+
+static inline void pte_chain_unlock(struct page *page)
+{
+	clear_bit(PG_chainlock, &page->flags);
+}
 
 /*
  * The zone field is never updated after free_area_init_core()

diff -Nru a/mm/filemap.c b/mm/filemap.c
@@ -233,8 +233,14 @@
 
 static void truncate_complete_page(struct page *page)
 {
-	/* Leave it on the LRU if it gets converted into anonymous buffers */
-	if (!page->pte_chain && (!page->buffers || do_flushpage(page, 0)))
+	/*
+	 * Leave it on the LRU if it gets converted into anonymous buffers
+	 * It is furthermore assumed that no process has mapped this page.
+	 */
+	if (page->pte_chain)
+		BUG();
+
+	if (!page->buffers || do_flushpage(page, 0))
 		lru_cache_del(page);
 
 	/*

115 lines not on this page · raw message

diff -Nru a/mm/swap.c b/mm/swap.c
@@ -104,6 +104,7 @@
 	}
 
 	/* Make sure the page really is reclaimable. */
+	pte_chain_lock(page);
 	if (!page->mapping || PageDirty(page) || page->pte_chain ||
 			page->buffers || page_count(page) > 1)
 		deactivate_page_nolock(page);

@@ -119,6 +120,7 @@
 			add_page_to_inactive_clean_list(page);
 		}
 	}
+	pte_chain_unlock(page);
 }
 
 /*

diff -Nru a/mm/vmscan.c b/mm/vmscan.c
@@ -47,6 +47,7 @@
 	page->age -= min(PAGE_AGE_DECL, (int)page->age);
 }
 
+/* Must be called with page's pte_chain_lock held. */
 static inline int page_mapping_inuse(struct page * page)
 {
 	struct address_space * mapping = page->mapping;

@@ -108,14 +109,21 @@
 		}
 
 		/* Page cannot be reclaimed ?  Move to inactive_dirty list. */
+		pte_chain_lock(page);
 		if (unlikely(page->pte_chain || page->buffers ||
 				PageReferenced(page) || PageDirty(page) ||
 				page_count(page) > 1 || TryLockPage(page))) {
 			del_page_from_inactive_clean_list(page);
 			add_page_to_inactive_dirty_list(page);
+			pte_chain_unlock(page);
 			continue;
 		}
 
+		/*
+		 * From here until reaching either the bottom of the loop
+		 * or found_page: the pte_chain_lock is held.
+		 */
+
 		/* OK, remove the page from the caches. */
                 if (PageSwapCache(page)) {
 			entry.val = page->index;

@@ -132,6 +140,7 @@
 		printk(KERN_ERR "VM: reclaim_page, found unknown page\n");
 		list_del(page_lru);
 		zone->inactive_clean_pages--;
+		pte_chain_unlock(page);
 		UnlockPage(page);
 	}
 	spin_unlock(&pagecache_lock);

@@ -141,6 +150,7 @@
 
 found_page:
 	del_page_from_inactive_clean_list(page);
+	pte_chain_unlock(page);
 	spin_unlock(&pagecache_lock);
 	spin_unlock(&pagemap_lru_lock);
 	if (entry.val)

@@ -243,13 +253,16 @@
 		 * The page is in active use or really unfreeable. Move to
 		 * the active list and adjust the page age if needed.
 		 */
+		pte_chain_lock(page);
 		if (page_referenced(page) && page_mapping_inuse(page) &&
 				!page_over_rsslimit(page)) {
 			del_page_from_inactive_dirty_list(page);
 			add_page_to_active_list(page);
 			page->age = max((int)page->age, PAGE_AGE_START);
+			pte_chain_unlock(page);
 			continue;
 		}
+		pte_chain_unlock(page);
 
 		/*
 		 * Page is being freed, don't worry about it.

@@ -270,8 +283,10 @@
 		 *
 		 * XXX: implement swap clustering ?
 		 */
+		pte_chain_lock(page);
 		if (page->pte_chain && !page->mapping && !page->buffers) {
 			page_cache_get(page);
+			pte_chain_unlock(page);
 			spin_unlock(&pagemap_lru_lock);
 			if (!add_to_swap(page)) {
 				activate_page(page);

@@ -282,24 +297,28 @@
 			}
 			page_cache_release(page);
 			spin_lock(&pagemap_lru_lock);
-		}
+		} else
+			pte_chain_unlock(page);
 
 		/*
 		 * The page is mapped into the page tables of one or more
 		 * processes. Try to unmap it here.
 		 */
+		pte_chain_lock(page);
 		if (page->pte_chain) {
 			switch (try_to_unmap(page)) {
 				case SWAP_ERROR:
 				case SWAP_FAIL:
 					goto page_active;
 				case SWAP_AGAIN:
+					pte_chain_unlock(page);
 					UnlockPage(page);
 					continue;
 				case SWAP_SUCCESS:
 					; /* try to free the page below */
 			}
 		}
+		pte_chain_unlock(page);
 
 		if (PageDirty(page) && page->mapping) {
 			/*

@@ -386,10 +405,12 @@
 		 * This test is not safe from races, but only the one
 		 * in reclaim_page() needs to be.
 		 */
+		pte_chain_lock(page);
 		if (page->mapping && !PageDirty(page) && !page->pte_chain &&
 				page_count(page) == 1) {
 			del_page_from_inactive_dirty_list(page);
 			add_page_to_inactive_clean_list(page);
+			pte_chain_unlock(page);
 			UnlockPage(page);
 			cleaned_pages++;
 		} else {

@@ -401,6 +422,7 @@
 page_active:
 			del_page_from_inactive_dirty_list(page);
 			add_page_to_active_list(page);
+			pte_chain_unlock(page);
 			UnlockPage(page);
 		}
 	}

@@ -471,8 +493,12 @@
 		 * If the object the page is in is not in use we don't
 		 * bother with page aging.  If the page is touched again
 		 * while on the inactive_clean list it'll be reactivated.
+		 * From here until the end of the current iteration
+		 * the pte_chain_lock is held.
 		 */
+		pte_chain_lock(page);
 		if (!page_mapping_inuse(page)) {
+			pte_chain_unlock(page);
 			drop_page(page);
 			continue;
 		}

@@ -496,9 +522,12 @@
 			list_add(page_lru, &zone->active_list);
 		} else {
 			deactivate_page_nolock(page);
-			if (++nr_deactivated > target)
-				break;
+			if (++nr_deactivated > target) {
+				pte_chain_unlock(page);
+				goto done;
+			}
 		}
+		pte_chain_unlock(page);
 
 		/* Low latency reschedule point */
 		if (current->need_resched) {

@@ -507,6 +536,8 @@
 			spin_lock(&pagemap_lru_lock);
 		}
 	}
+
+done:
 	spin_unlock(&pagemap_lru_lock);
 
 	return nr_deactivated;