Skip to contentSkip to conversation

disable interrupts while holding pagemap_lru_lock

PATCH In review Andrew Morton 2 participants 2 messages Aug 11, 2002

Subsystems (5)
Memory management MGLRU Page cache MM reclaim +1 more subsystems
lore
Share & tools
lore

Andrew Morton

author
#0 lore raw

PATCH 11/21 disable interrupts while holding pagemap_lru_lock

It is expensive for a CPU to take an interrupt while holding the page LRU lock, because other CPUs will pile up on the lock while the interrupt runs.

Disabling interrupts while holding the lock reduces contention by an additional 30% on 4-way. This is when the only source of interrupts is disk completion. The improvement will be higher with more CPUs and it will be higher if there is networking happening.

The maximum hold time of this lock is 17 microseconds on 500 MHx PIII, which is well inside the kernel's maximum interrupt latency (which was 100 usecs when I last looked, a year ago).

This optimisation is not needed on uniprocessor, but the patch disables IRQs while holding pagemap_lru_lock anyway, so it becomes an irq-safe spinlock, and pages can be moved from the LRU in interrupt context.

pagemap_lru_lock has been renamed to _pagemap_lru_lock to pick up any missed uses, and to reliably break any out-of-tree patches which may be using the old semantics.

 include/linux/swap.h |    2 +-
 mm/filemap.c         |    2 +-
 mm/swap.c            |   28 ++++++++++++++--------------
 mm/vmscan.c          |   28 ++++++++++++++--------------
 4 files changed, 30 insertions(+), 30 deletions(-)
mm/filemap.c renamed from mm/filemap.c~lru-lock-irq-off +1 -1
@@ -62,7 +62,7 @@
  *      ->inode_lock		(__mark_inode_dirty)
  *        ->sb_lock		(fs/fs-writeback.c)
  */
-spinlock_t pagemap_lru_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
+spinlock_t _pagemap_lru_lock __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED;
 
 /*
  * Remove a page from the page cache and free it. Caller has to make
mm/swap.c renamed from mm/swap.c~lru-lock-irq-off +14 -14
@@ -41,9 +41,9 @@ static inline void activate_page_nolock(
  */
 void activate_page(struct page * page)
 {
-	spin_lock(&pagemap_lru_lock);
+	spin_lock_irq(&_pagemap_lru_lock);
 	activate_page_nolock(page);
-	spin_unlock(&pagemap_lru_lock);
+	spin_unlock_irq(&_pagemap_lru_lock);
 }
 
 /**

@@ -53,10 +53,10 @@ void activate_page(struct page * page)
 void lru_cache_add(struct page * page)
 {
 	if (!PageLRU(page)) {
-		spin_lock(&pagemap_lru_lock);
+		spin_lock_irq(&_pagemap_lru_lock);
 		if (!TestSetPageLRU(page))
 			add_page_to_inactive_list(page);
-		spin_unlock(&pagemap_lru_lock);
+		spin_unlock_irq(&_pagemap_lru_lock);
 	}
 }
 

@@ -83,9 +83,9 @@ void __lru_cache_del(struct page * page)
  */
 void lru_cache_del(struct page * page)
 {
-	spin_lock(&pagemap_lru_lock);
+	spin_lock_irq(&_pagemap_lru_lock);
 	__lru_cache_del(page);
-	spin_unlock(&pagemap_lru_lock);
+	spin_unlock_irq(&_pagemap_lru_lock);
 }
 
 /*

@@ -116,7 +116,7 @@ void __pagevec_release(struct pagevec *p
 			continue;
 
 		if (!lock_held) {
-			spin_lock(&pagemap_lru_lock);
+			spin_lock_irq(&_pagemap_lru_lock);
 			lock_held = 1;
 		}
 

@@ -130,7 +130,7 @@ void __pagevec_release(struct pagevec *p
 			pagevec_add(&pages_to_free, page);
 	}
 	if (lock_held)
-		spin_unlock(&pagemap_lru_lock);
+		spin_unlock_irq(&_pagemap_lru_lock);
 
 	pagevec_free(&pages_to_free);
 	pagevec_init(pvec);

@@ -175,14 +175,14 @@ void pagevec_deactivate_inactive(struct 
 		if (!lock_held) {
 			if (PageActive(page) || !PageLRU(page))
 				continue;
-			spin_lock(&pagemap_lru_lock);
+			spin_lock_irq(&_pagemap_lru_lock);
 			lock_held = 1;
 		}
 		if (!PageActive(page) && PageLRU(page))
 			list_move(&page->lru, &inactive_list);
 	}
 	if (lock_held)
-		spin_unlock(&pagemap_lru_lock);
+		spin_unlock_irq(&_pagemap_lru_lock);
 	__pagevec_release(pvec);
 }
 

@@ -194,7 +194,7 @@ void __pagevec_lru_add(struct pagevec *p
 {
 	int i;
 
-	spin_lock(&pagemap_lru_lock);
+	spin_lock_irq(&_pagemap_lru_lock);
 	for (i = 0; i < pagevec_count(pvec); i++) {
 		struct page *page = pvec->pages[i];
 

@@ -202,7 +202,7 @@ void __pagevec_lru_add(struct pagevec *p
 			BUG();
 		add_page_to_inactive_list(page);
 	}
-	spin_unlock(&pagemap_lru_lock);
+	spin_unlock_irq(&_pagemap_lru_lock);
 	pagevec_release(pvec);
 }
 

@@ -214,7 +214,7 @@ void __pagevec_lru_del(struct pagevec *p
 {
 	int i;
 
-	spin_lock(&pagemap_lru_lock);
+	spin_lock_irq(&_pagemap_lru_lock);
 	for (i = 0; i < pagevec_count(pvec); i++) {
 		struct page *page = pvec->pages[i];
 

@@ -225,7 +225,7 @@ void __pagevec_lru_del(struct pagevec *p
 		else
 			del_page_from_inactive_list(page);
 	}
-	spin_unlock(&pagemap_lru_lock);
+	spin_unlock_irq(&_pagemap_lru_lock);
 	pagevec_release(pvec);
 }
mm/vmscan.c renamed from mm/vmscan.c~lru-lock-irq-off +14 -14
@@ -284,7 +284,7 @@ shrink_cache(int nr_pages, zone_t *class
 
 	pagevec_init(&pvec);
 
-	spin_lock(&pagemap_lru_lock);
+	spin_lock_irq(&_pagemap_lru_lock);
 	while (max_scan > 0 && nr_pages > 0) {
 		struct page *page;
 		int n = 0;

@@ -307,7 +307,7 @@ shrink_cache(int nr_pages, zone_t *class
 			page_cache_get(page);
 			n++;
 		}
-		spin_unlock(&pagemap_lru_lock);
+		spin_unlock_irq(&_pagemap_lru_lock);
 
 		if (list_empty(&page_list))
 			goto done;

@@ -321,7 +321,7 @@ shrink_cache(int nr_pages, zone_t *class
 		if (nr_pages <= 0 && list_empty(&page_list))
 			goto done;
 
-		spin_lock(&pagemap_lru_lock);
+		spin_lock_irq(&_pagemap_lru_lock);
 		/*
 		 * Put back any unfreeable pages.
 		 */

@@ -335,13 +335,13 @@ shrink_cache(int nr_pages, zone_t *class
 			else
 				add_page_to_inactive_list(page);
 			if (!pagevec_add(&pvec, page)) {
-				spin_unlock(&pagemap_lru_lock);
+				spin_unlock_irq(&_pagemap_lru_lock);
 				__pagevec_release(&pvec);
-				spin_lock(&pagemap_lru_lock);
+				spin_lock_irq(&_pagemap_lru_lock);
 			}
 		}
   	}
-	spin_unlock(&pagemap_lru_lock);
+	spin_unlock_irq(&_pagemap_lru_lock);
 done:
 	pagevec_release(&pvec);
 	return nr_pages;	

@@ -374,7 +374,7 @@ static /* inline */ void refill_inactive
 	struct page *page;
 	struct pagevec pvec;
 
-	spin_lock(&pagemap_lru_lock);
+	spin_lock_irq(&_pagemap_lru_lock);
 	while (nr_pages && !list_empty(&active_list)) {
 		page = list_entry(active_list.prev, struct page, lru);
 		prefetchw_prev_lru_page(page, &active_list, flags);

@@ -384,7 +384,7 @@ static /* inline */ void refill_inactive
 		list_move(&page->lru, &l_hold);
 		nr_pages--;
 	}
-	spin_unlock(&pagemap_lru_lock);
+	spin_unlock_irq(&_pagemap_lru_lock);
 
 	while (!list_empty(&l_hold)) {
 		page = list_entry(l_hold.prev, struct page, lru);

@@ -406,7 +406,7 @@ static /* inline */ void refill_inactive
 	}
 
 	pagevec_init(&pvec);
-	spin_lock(&pagemap_lru_lock);
+	spin_lock_irq(&_pagemap_lru_lock);
 	while (!list_empty(&l_inactive)) {
 		page = list_entry(l_inactive.prev, struct page, lru);
 		prefetchw_prev_lru_page(page, &l_inactive, flags);

@@ -416,9 +416,9 @@ static /* inline */ void refill_inactive
 			BUG();
 		list_move(&page->lru, &inactive_list);
 		if (!pagevec_add(&pvec, page)) {
-			spin_unlock(&pagemap_lru_lock);
+			spin_unlock_irq(&_pagemap_lru_lock);
 			__pagevec_release(&pvec);
-			spin_lock(&pagemap_lru_lock);
+			spin_lock_irq(&_pagemap_lru_lock);
 		}
 	}
 	while (!list_empty(&l_active)) {

@@ -429,12 +429,12 @@ static /* inline */ void refill_inactive
 		BUG_ON(!PageActive(page));
 		list_move(&page->lru, &active_list);
 		if (!pagevec_add(&pvec, page)) {
-			spin_unlock(&pagemap_lru_lock);
+			spin_unlock_irq(&_pagemap_lru_lock);
 			__pagevec_release(&pvec);
-			spin_lock(&pagemap_lru_lock);
+			spin_lock_irq(&_pagemap_lru_lock);
 		}
 	}
-	spin_unlock(&pagemap_lru_lock);
+	spin_unlock_irq(&_pagemap_lru_lock);
 	pagevec_release(&pvec);
 
 	mod_page_state(nr_active, -pgdeactivate);
include/linux/swap.h renamed from include/linux/swap.h~lru-lock-irq-off +1 -1
@@ -212,7 +212,7 @@ extern struct swap_list_t swap_list;
 asmlinkage long sys_swapoff(const char *);
 asmlinkage long sys_swapon(const char *, int);
 
-extern spinlock_t pagemap_lru_lock;
+extern spinlock_t _pagemap_lru_lock;
 
 extern void FASTCALL(mark_page_accessed(struct page *));

.

#1 lore raw
Quoted 3 lines · Andrew Morton

This optimisation is not needed on uniprocessor, but the patch disables IRQs while holding pagemap_lru_lock anyway, so it becomes an irq-safe spinlock, and pages can be moved from the LRU in interrupt context.

It does cleanup slablru in UP. It simplifies the patch getting rid of the #ifdef when adding page(s) to the slab caches. It also cleans up the code used when we want to remove a slab from a cache.

Thanks
Ed Tomlinson