mm: add free_hot_cold_page_list helper

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28)
function old new delta
free_hot_cold_page_list - 264 +264

get_page_from_freelist                      2129    2132      +3
__pagevec_free                               243     239      -4
split_free_page                              380     373      -7

release_pages 606 510 -96
free_page_list 188 - -188

index cb40892..dd7b9cc 100644

include/linux/gfp.h +1 -0

@@ -358,6 +358,7 @@ void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
 extern void __free_pages(struct page *page, unsigned int order);
 extern void free_pages(unsigned long addr, unsigned int order);
 extern void free_hot_cold_page(struct page *page, int cold);
+extern void free_hot_cold_page_list(struct list_head *list, int cold);
 
 #define __free_page(page) __free_pages((page), 0)
 #define free_page(addr) free_pages((addr), 0)

mm/page_alloc.c +12 -0

index 1dbcf88..af486e4 100644
@@ -1209,6 +1209,18 @@ out:
 	local_irq_restore(flags);
 }
 
+void free_hot_cold_page_list(struct list_head *list, int cold)
+{
+	struct page *page, *next;
+
+	list_for_each_entry_safe(page, next, list, lru) {
+		trace_mm_pagevec_free(page, cold);
+		free_hot_cold_page(page, cold);
+	}
+
+	INIT_LIST_HEAD(list);
+}
+
 /*
  * split_page takes a non-compound higher-order page, and splits it into
  * n (1<<order) sub-pages: page[0..n]

mm/swap.c +1 -2

index 3a442f1..b9138c7 100644
@@ -562,11 +562,10 @@ int lru_add_drain_all(void)
 void release_pages(struct page **pages, int nr, int cold)
 {
 	int i;
-	struct pagevec pages_to_free;
+	LIST_HEAD(pages_to_free);
 	struct zone *zone = NULL;
 	unsigned long uninitialized_var(flags);
 
-	pagevec_init(&pages_to_free, cold);
 	for (i = 0; i < nr; i++) {
 		struct page *page = pages[i];
 

@@ -597,19 +596,12 @@ void release_pages(struct page **pages, int nr, int cold)
 			del_page_from_lru(zone, page);
 		}

Quoted 6 lines · On Fri, 29 Jul 2011 11:58:37 +0400 Konstantin…

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

Quoted 9 lines

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28)
function old new delta
free_hot_cold_page_list - 264 +264

get_page_from_freelist                      2129    2132      +3
__pagevec_free                               243     239      -4
split_free_page                              380     373      -7

release_pages 606 510 -96
free_page_list 188 - -188

It saves a total of 150 bytes for me.

Quoted 54 lines

index cb40892..dd7b9cc 100644

include/linux/gfp.h +1 -0

@@ -358,6 +358,7 @@ void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
  extern void __free_pages(struct page *page, unsigned int order);
  extern void free_pages(unsigned long addr, unsigned int order);
  extern void free_hot_cold_page(struct page *page, int cold);
+extern void free_hot_cold_page_list(struct list_head *list, int cold);
 
  #define __free_page(page) __free_pages((page), 0)
  #define free_page(addr) free_pages((addr), 0)

mm/page_alloc.c +12 -0

index 1dbcf88..af486e4 100644
@@ -1209,6 +1209,18 @@ out:
  	local_irq_restore(flags);
  }
 
+void free_hot_cold_page_list(struct list_head *list, int cold)
+{
+	struct page *page, *next;
+
+	list_for_each_entry_safe(page, next, list, lru) {
+		trace_mm_pagevec_free(page, cold);
+		free_hot_cold_page(page, cold);
+	}
+
+	INIT_LIST_HEAD(list);
+}
+
  /*
   * split_page takes a non-compound higher-order page, and splits it into
   * n (1<<order) sub-pages: page[0..n]

mm/swap.c +1 -2

index 3a442f1..b9138c7 100644
@@ -562,11 +562,10 @@ int lru_add_drain_all(void)
  void release_pages(struct page **pages, int nr, int cold)
  {
  	int i;
-	struct pagevec pages_to_free;
+	LIST_HEAD(pages_to_free);
  	struct zone *zone = NULL;
  	unsigned long uninitialized_var(flags);
 
-	pagevec_init(&pages_to_free, cold);
  	for (i = 0; i<  nr; i++) {
  		struct page *page = pages[i];
 

@@ -597,19 +596,12 @@ void release_pages(struct page **pages, int nr, int cold)
  			del_page_from_lru(zone, page);
  		}

mm/swap.c void release_pages(struct page **pages, int nr, int cold) · line 599

Quoted 10 lines

mm/swap.c +1 -8

 
-		if (!pagevec_add(&pages_to_free, page)) {
-			if (zone) {
-				spin_unlock_irqrestore(&zone->lru_lock, flags);
-				zone = NULL;
-			}
-			__pagevec_free(&pages_to_free);
-			pagevec_reinit(&pages_to_free);
-  		}
+		list_add_tail(&page->lru,&pages_to_free);

There's a potential problem here with cache longevity. If release_pages() is called with a large number of pages then the current code's approach of freeing pages 16-at-a-time will hopefully cause those pageframes to still be in CPU cache when we get to actually freeing them.

But after this change, we free all the pages in a single operation right at the end, which adds risk that we'll have to reload all their pageframes into CPU cache again.

That'll only be a problem if release_pages() _is_ called with a large number of pages. And manipulating large numbers of pages represents a lot of work, so the additional work from one cachemiss per page will presumably be tiny.

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28) function                                     old     new   delta free_hot_cold_page_list                        -     264    +264 get_page_from_freelist                      2129    2132      +3 __pagevec_free                               243     239      -4 split_free_page                              380     373      -7 release_pages                                606     510     -96 free_page_list                               188       -    -188

Signed-off-by Konstantin Khlebnikov <[email protected]>

---
 include/linux/gfp.h |    1 +
 mm/page_alloc.c     |   12 ++++++++++++
 mm/swap.c           |   14 +++-----------
 mm/vmscan.c         |   20 +-------------------

 4 files changed, 17 insertions(+), 30 deletions(-)

include/linux/gfp.h +1 -0

index cb40892..dd7b9cc 100644
@@ -358,6 +358,7 @@ void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
  extern void __free_pages(struct page *page, unsigned int order);
  extern void free_pages(unsigned long addr, unsigned int order);
  extern void free_hot_cold_page(struct page *page, int cold);
+extern void free_hot_cold_page_list(struct list_head *list, int cold);
 
  #define __free_page(page) __free_pages((page), 0)
  #define free_page(addr) free_pages((addr), 0)

mm/page_alloc.c +0 -0

index 1dbcf88..af486e4 100644

Quoted 40 lines · Konstantin Khlebnikov

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28)
function old new delta
free_hot_cold_page_list - 264 +264

get_page_from_freelist                      2129    2132      +3
 pagevec_free                               243     239      -4
split_free_page                              380     373      -7
release_pages                                606     510     -96
free_page_list                               188       -    -188

Signed-off-by Konstantin Khlebnikov <[email protected]>

---
 include/linux/gfp.h |    1 +
 mm/page_alloc.c     |   12 ++++++++++++
 mm/swap.c           |   14 +++-----------
 mm/vmscan.c         |   20 +-------------------
 4 files changed, 17 insertions(+), 30 deletions(-)

include/linux/gfp.h +1 -0

index cb40892..dd7b9cc 100644
@@ -358,6 +358,7 @@ void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
 extern void  free_pages(struct page *page, unsigned int order);
 extern void free_pages(unsigned long addr, unsigned int order);
 extern void free_hot_cold_page(struct page *page, int cold);
+extern void free_hot_cold_page_list(struct list_head *list, int cold);
 
 #define  free_page(page)  free_pages((page), 0)
 #define free_page(addr) free_pages((addr), 0)

mm/page_alloc.c +0 -0

index 1dbcf88..af486e4 100644

mm/page_alloc.c out: · line 1217

Quoted 9 lines

mm/page_alloc.c +6 -0

@@ -1209,6 +1209,18 @@ out:
       local_irq_restore(flags);
 }
 
+void free_hot_cold_page_list(struct list_head *list, int cold)
+{
+       struct page *page, *next;
+
+       list_for_each_entry_safe(page, next, list, lru) {
+               trace_mm_pagevec_free(page, cold);

I understand you want to minimize changes without breaking current ABI with trace tools.
But apparently, It's not a pagvec_free. It just hurts readability. As I take a look at the code, mm_pagevec_free isn't related to pagevec but I guess it can represent 0-order pages free because 0-order pages are freed only by pagevec until now.
So, how about renaming it with mm_page_free or mm_page_free_zero_order? If you do, you need to do s/MM_PAGEVEC_FREE/MM_FREE_FREE/g in trace-pagealloc-postprocess.pl.

mm/page_alloc.c out: · line 1221

Quoted 4 lines

mm/page_alloc.c +4 -0

+               free_hot_cold_page(page, cold);
+       }
+
+       INIT_LIST_HEAD(list);

Why do we need it?

Quoted 69 lines · On Mon, 29 Aug 2011 16:48:46 +0900 Minchan Kim<[email protected]> wrote:

Quoted 40 lines · Konstantin Khlebnikov

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28)
function old new delta
free_hot_cold_page_list - 264 +264

get_page_from_freelist                      2129    2132      +3
  pagevec_free                               243     239      -4
split_free_page                              380     373      -7

release_pages 606 510 -96
free_page_list 188 - -188

Signed-off-by Konstantin Khlebnikov<[email protected]>

---
  include/linux/gfp.h |    1 +
  mm/page_alloc.c     |   12 ++++++++++++
  mm/swap.c           |   14 +++-----------
  mm/vmscan.c         |   20 +-------------------
  4 files changed, 17 insertions(+), 30 deletions(-)

include/linux/gfp.h +1 -0

index cb40892..dd7b9cc 100644
@@ -358,6 +358,7 @@ void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
  extern void  free_pages(struct page *page, unsigned int order);
  extern void free_pages(unsigned long addr, unsigned int order);
  extern void free_hot_cold_page(struct page *page, int cold);
+extern void free_hot_cold_page_list(struct list_head *list, int cold);
 
  #define  free_page(page)  free_pages((page), 0)
  #define free_page(addr) free_pages((addr), 0)

mm/page_alloc.c +0 -0

index 1dbcf88..af486e4 100644

mm/page_alloc.c out: · line 1217

Quoted 9 lines

mm/page_alloc.c +6 -0

@@ -1209,6 +1209,18 @@ out:
        local_irq_restore(flags);
  }
 
+void free_hot_cold_page_list(struct list_head *list, int cold)
+{
+       struct page *page, *next;
+
+       list_for_each_entry_safe(page, next, list, lru) {
+               trace_mm_pagevec_free(page, cold);

I understand you want to minimize changes without breaking current ABI with trace tools.
But apparently, It's not a pagvec_free. It just hurts readability. As I take a look at the code, mm_pagevec_free isn't related to pagevec but I guess it can represent 0-order pages free because 0-order pages are freed only by pagevec until now.
So, how about renaming it with mm_page_free or mm_page_free_zero_order? If you do, you need to do s/MM_PAGEVEC_FREE/MM_FREE_FREE/g in trace-pagealloc-postprocess.pl.

Quoted 4 lines

mm/page_alloc.c +4 -0

+               free_hot_cold_page(page, cold);
+       }
+
+       INIT_LIST_HEAD(list);

Quoted 1 line

Why do we need it?

My email has been horrid for a couple of months (fixed now), so I might have missed any reply to Minchin's review comments?

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from the list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28)
function old new delta
free_hot_cold_page_list - 264 +264

get_page_from_freelist                      2129    2132      +3
__pagevec_free                               243     239      -4
split_free_page                              380     373      -7

release_pages 606 510 -96
free_page_list 188 - -188

Quoted 6 lines · On Tue, 01 Nov 2011 11:45:02 +0300 Konstantin…

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from the list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

Quoted 8 lines

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28)
function old new delta
free_hot_cold_page_list - 264 +264

get_page_from_freelist                      2129    2132      +3
__pagevec_free                               243     239      -4
split_free_page                              380     373      -7

release_pages 606 510 -96
free_page_list 188 - -188

Here's what you changed:

mm/page_alloc.c renamed from mm/page_alloc.c~mm-add-free_hot_cold_page_list-helper-v2 +3 -2

@@ -1210,6 +1210,9 @@ out:
  	local_irq_restore(flags);
  }
 
+/*
+ * Free a list of 0-order pages
+ */
  void free_hot_cold_page_list(struct list_head *list, int cold)
  {
  	struct page *page, *next;

@@ -1218,8 +1221,6 @@ void free_hot_cold_page_list(struct list
  		trace_mm_pagevec_free(page, cold);
  		free_hot_cold_page(page, cold);
  	}
-
-	INIT_LIST_HEAD(list);
  }
 
  /*

_

However I can't find any sign that you addressed Minchin's original review comment regarding free_hot_cold_page_list():

: I understand you want to minimize changes without breaking current ABI : with trace tools.
: But apparently, It's not a pagvec_free. It just hurts readability. : As I take a look at the code, mm_pagevec_free isn't related to pagevec : but I guess it can represent 0-order pages free because 0-order pages : are freed only by pagevec until now.
: So, how about renaming it with mm_page_free or mm_page_free_zero_order? : If you do, you need to do s/MM_PAGEVEC_FREE/MM_FREE_FREE/g in : trace-pagealloc-postprocess.pl.

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from the list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

Quoted 3 lines · Konstantin Khlebnikov

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from the list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

Sorry for not speaking up sooner, but I do like this patch very much (and I'm content with your trace compatibility choice - whatever).

Not so much in itself, but because it then allows a further patch (mainly to mm/vmscan.c) to remove two levels of pagevec, reducing its deepest stack by around 240 bytes.

I have that patch, but keep putting off sending it in, because I want to show a reclaim stack overflow that it prevents, but the new avoidance of writeback in direct reclaim makes that harder to demonstrate. Damn!

One question on your patch: where you have release_pages() doing

Review

Quoted 1 line

(quoted diff) +1 -0

+		list_add_tail(&page->lru,&pages_to_free);

That seems reasonable, but given that __pagevec_free() proceeds by

	while (--i>= 0) {

, starting from the far end of the pagevec (the most recently added struct page, the most likely to be hot), wouldn't you reproduce existing behaviour more accurately by a simple list_add()?

Or have I got that back to front? If so, a comment on the list_add_tail() would help me to remember why - thanks.

Hugh

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from the list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28) function                                     old     new   delta free_hot_cold_page_list                        -     264    +264 get_page_from_freelist                      2129    2132      +3 __pagevec_free                               243     239      -4 split_free_page                              380     373      -7 release_pages                                606     510     -96 free_page_list                               188       -    -188

v2: Remove list reinititialization.
v3: Always free pages in reverse order.
   The most recently added struct page, the most likely to be hot.

Signed-off-by Konstantin Khlebnikov <[email protected]>

This patch adds helper free_hot_cold_page_list() to free list of 0-order pages. It frees pages directly from the list without temporary page-vector. It also calls trace_mm_pagevec_free() to simulate pagevec_free() behaviour.

bloat-o-meter:

add/remove: 1/1 grow/shrink: 1/3 up/down: 267/-295 (-28)
function old new delta
free_hot_cold_page_list - 264 +264

get_page_from_freelist                      2129    2132      +3
__pagevec_free                               243     239      -4
split_free_page                              380     373      -7

release_pages 606 510 -96
free_page_list 188 - -188

v2: Remove list reinititialization.
v3: Always free pages in reverse order.
The most recently added struct page, the most likely to be hot.

Signed-off-by Konstantin Khlebnikov <[email protected]>

---
 include/linux/gfp.h |    1 +
 mm/page_alloc.c     |   13 +++++++++++++
 mm/swap.c           |   14 +++-----------
 mm/vmscan.c         |   20 +-------------------
 4 files changed, 18 insertions(+), 30 deletions(-)

include/linux/gfp.h +1 -0

index 3a76faf..6562958 100644
@@ -358,6 +358,7 @@ void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask);
 extern void __free_pages(struct page *page, unsigned int order);
 extern void free_pages(unsigned long addr, unsigned int order);
 extern void free_hot_cold_page(struct page *page, int cold);
+extern void free_hot_cold_page_list(struct list_head *list, int cold);
 
 #define __free_page(page) __free_pages((page), 0)
 #define free_page(addr) free_pages((addr), 0)

mm/page_alloc.c +13 -0

index 9dd443d..5093114 100644
@@ -1211,6 +1211,19 @@ out:
 }
 
 /*
+ * Free a list of 0-order pages
+ */
+void free_hot_cold_page_list(struct list_head *list, int cold)
+{
+	struct page *page, *next;
+
+	list_for_each_entry_safe(page, next, list, lru) {
+		trace_mm_pagevec_free(page, cold);
+		free_hot_cold_page(page, cold);
+	}
+}
+
+/*
  * split_page takes a non-compound higher-order page, and splits it into
  * n (1<<order) sub-pages: page[0..n]
  * Each sub-page must be freed individually.

mm/swap.c +3 -11

index a91caf7..67a09a6 100644
@@ -585,11 +585,10 @@ int lru_add_drain_all(void)
 void release_pages(struct page **pages, int nr, int cold)
 {
 	int i;
-	struct pagevec pages_to_free;
+	LIST_HEAD(pages_to_free);
 	struct zone *zone = NULL;
 	unsigned long uninitialized_var(flags);
 
-	pagevec_init(&pages_to_free, cold);
 	for (i = 0; i < nr; i++) {
 		struct page *page = pages[i];
 

@@ -620,19 +619,12 @@ void release_pages(struct page **pages, int nr, int cold)
 			del_page_from_lru(zone, page);
 		}
 
-		if (!pagevec_add(&pages_to_free, page)) {
-			if (zone) {
-				spin_unlock_irqrestore(&zone->lru_lock, flags);
-				zone = NULL;
-			}
-			__pagevec_free(&pages_to_free);
-			pagevec_reinit(&pages_to_free);
-  		}
+		list_add(&page->lru, &pages_to_free);
 	}
 	if (zone)
 		spin_unlock_irqrestore(&zone->lru_lock, flags);
 
-	pagevec_free(&pages_to_free);
+	free_hot_cold_page_list(&pages_to_free, cold);
 }
 EXPORT_SYMBOL(release_pages);
 

mm/vmscan.c +0 -18

index a1893c0..f4be53d 100644
@@ -728,24 +728,6 @@ static enum page_references page_check_references(struct page *page,
 	return PAGEREF_RECLAIM;
 }
 
-static noinline_for_stack void free_page_list(struct list_head *free_pages)
-{
-	struct pagevec freed_pvec;
-	struct page *page, *tmp;
-
-	pagevec_init(&freed_pvec, 1);
-
-	list_for_each_entry_safe(page, tmp, free_pages, lru) {
-		list_del(&page->lru);
-		if (!pagevec_add(&freed_pvec, page)) {
-			__pagevec_free(&freed_pvec);
-			pagevec_reinit(&freed_pvec);
-		}
-	}
-
-	pagevec_free(&freed_pvec);
-}
-
 /*
  * shrink_page_list() returns the number of reclaimed pages
  */

It not exported and now nobody use it.

Signed-off-by Konstantin Khlebnikov <[email protected]>

It not exported and now nobody use it.

Signed-off-by Konstantin Khlebnikov <[email protected]>

---
 include/linux/pagevec.h |    7 -------
 mm/page_alloc.c         |   10 ----------
 2 files changed, 0 insertions(+), 17 deletions(-)

include/linux/pagevec.h +0 -7

index bab82f4..ed17024 100644
@@ -21,7 +21,6 @@ struct pagevec {
 };
 
 void __pagevec_release(struct pagevec *pvec);
-void __pagevec_free(struct pagevec *pvec);
 void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru);
 void pagevec_strip(struct pagevec *pvec);
 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,

@@ -67,12 +66,6 @@ static inline void pagevec_release(struct pagevec *pvec)
 		__pagevec_release(pvec);
 }
 
-static inline void pagevec_free(struct pagevec *pvec)
-{
-	if (pagevec_count(pvec))
-		__pagevec_free(pvec);
-}
-
 static inline void __pagevec_lru_add_anon(struct pagevec *pvec)
 {
 	____pagevec_lru_add(pvec, LRU_INACTIVE_ANON);

mm/page_alloc.c +0 -10

index 5093114..0562d85 100644
@@ -2341,16 +2341,6 @@ unsigned long get_zeroed_page(gfp_t gfp_mask)
 }
 EXPORT_SYMBOL(get_zeroed_page);
 
-void __pagevec_free(struct pagevec *pvec)
-{
-	int i = pagevec_count(pvec);
-
-	while (--i >= 0) {
-		trace_mm_pagevec_free(pvec->pages[i], pvec->cold);
-		free_hot_cold_page(pvec->pages[i], pvec->cold);
-	}
-}
-
 void __free_pages(struct page *page, unsigned int order)
 {
 	if (put_page_testzero(page)) {

Rename mm_page_free_direct into mm_page_free
and mm_pagevec_free into mm_page_free_batched

Since v2.6.33-5426-gc475dab kernel trigger mm_page_free_direct for all freed pages, not only for directly freed. So, let's name it properly.
For pages freed via page-list we also trigger mm_page_free_batched event.

Signed-off-by Konstantin Khlebnikov <[email protected]>

Rename page-free mm tracepoints.

Signed-off-by Konstantin Khlebnikov <[email protected]>