mm: fix some MADV_FREE issues

mm/huge_memory.c +2 -0

@@ -1404,6 +1401,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		set_pmd_at(mm, addr, pmd, orig_pmd);
 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
 	}
+
+	mark_page_lazyfree(page);
 	ret = true;
 out:
 	spin_unlock(ptl);

mm/swap.c +4 -4

-void deactivate_page(struct page *page)
-{
-	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
-		struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
+void mark_page_lazyfree(struct page *page)
+ {
+	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	    !PageUnevictable(page)) {

Quoted 9 lines · Shaohua Li

mm/huge_memory.c +2 -0

@@ -1404,6 +1401,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		set_pmd_at(mm, addr, pmd, orig_pmd);
 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
 	}
+
+	mark_page_lazyfree(page);
 	ret = true;
 out:
 	spin_unlock(ptl);

<snipped>

Quoted 10 lines

mm/swap.c +5 -4

-void deactivate_page(struct page *page)
-{
-	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
-		struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
+void mark_page_lazyfree(struct page *page)
+ {
+	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	    !PageUnevictable(page)) {
+		struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs);

mm/swap.c void deactivate_file_page(struct page *page) · line 664

Quoted 8 lines

mm/swap.c +2 -2

 		get_page(page);
 		if (!pagevec_add(pvec, page) || PageCompound(page))
-			pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
-		put_cpu_var(lru_deactivate_pvecs);
+			pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
+		put_cpu_var(lru_lazyfree_pvecs);
 	}
 }

You are not adding it but would you please try to fix or avoid flipping preempt count with page table lock hold?

Userspace indicates MADV_FREE pages could be freed without pageout, so it pretty much likes used once file pages. For such pages, we'd like to reclaim them once there is memory pressure. Also it might be unfair reclaiming MADV_FREE pages always before used once file pages and we definitively want to reclaim the pages before other anonymous and file pages.

To speed up MADV_FREE pages reclaim, we put the pages into LRU_INACTIVE_FILE list. The rationale is LRU_INACTIVE_FILE list is tiny nowadays and should be full of used once file pages. Reclaiming MADV_FREE pages will not have much interfere of anonymous and active file pages. And the inactive file pages and MADV_FREE pages will be reclaimed according to their age, so we don't reclaim too many MADV_FREE pages too. Putting the MADV_FREE pages into LRU_INACTIVE_FILE_LIST also means we can reclaim the pages without swap support. This idea is suggested by Johannes.

We also clear the pages SwapBacked flag to indicate they are MADV_FREE pages.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 include/linux/mm_inline.h     |  5 +++++
 include/linux/swap.h          |  2 +-
 include/linux/vm_event_item.h |  2 +-
 mm/huge_memory.c              |  5 ++---
 mm/madvise.c                  |  3 +--
 mm/swap.c                     | 50 ++++++++++++++++++++++++-------------------
 mm/vmstat.c                   |  1 +
 7 files changed, 39 insertions(+), 29 deletions(-)

include/linux/mm_inline.h +0 -0

index e030a68..fdded06 100644

include/linux/swap.h +0 -0

 extern void swap_setup(void);
 
 extern void add_page_to_unevictable_list(struct page *page);

include/linux/vm_event_item.h +1 -1

index 6aa1b6c..94e58da 100644
@@ -25,7 +25,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		FOR_ALL_ZONES(PGALLOC),
 		FOR_ALL_ZONES(ALLOCSTALL),
 		FOR_ALL_ZONES(PGSCAN_SKIP),
-		PGFREE, PGACTIVATE, PGDEACTIVATE,
+		PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
 		PGFAULT, PGMAJFAULT,
 		PGLAZYFREED,
 		PGREFILL,

mm/huge_memory.c +2 -3

index ecf569d..ddb9a94 100644
@@ -1391,9 +1391,6 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		ClearPageDirty(page);
 	unlock_page(page);
 
-	if (PageActive(page))
-		deactivate_page(page);
-
 	if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
 		orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
 			tlb->fullmm);

@@ -1404,6 +1401,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		set_pmd_at(mm, addr, pmd, orig_pmd);
 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
 	}
+
+	mark_page_lazyfree(page);
 	ret = true;
 out:
 	spin_unlock(ptl);

mm/madvise.c +1 -2

index c867d88..c24549e 100644
@@ -378,10 +378,9 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 			ptent = pte_mkclean(ptent);
 			ptent = pte_wrprotect(ptent);
 			set_pte_at(mm, addr, pte, ptent);
-			if (PageActive(page))
-				deactivate_page(page);
 			tlb_remove_tlb_entry(tlb, pte, addr);
 		}
+		mark_page_lazyfree(page);
 	}
 out:
 	if (nr_swap) {

mm/swap.c +1 -1

index c4910f1..69a7e9d 100644
@@ -46,7 +46,7 @@ int page_cluster;
 static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
-static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
+static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs);
 #ifdef CONFIG_SMP
 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
 #endif

mm/swap.c +24 -21

+			lru = LRU_INACTIVE_ANON;
+		}
 		SetPageActive(page);
 		lru += LRU_ACTIVE;
 		add_page_to_lru_list(page, lruvec, lru);

@@ -561,20 +566,21 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,
 }
 
 
-static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
+static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,
 			    void *arg)
 {
-	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
-		int file = page_is_file_cache(page);
-		int lru = page_lru_base_type(page);
+	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	    !PageUnevictable(page)) {
+		bool active = PageActive(page);
 
-		del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE);
+		del_page_from_lru_list(page, lruvec, LRU_INACTIVE_ANON + active);
 		ClearPageActive(page);
 		ClearPageReferenced(page);
-		add_page_to_lru_list(page, lruvec, lru);
+		ClearPageSwapBacked(page);
+		add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);
 
-		__count_vm_event(PGDEACTIVATE);
-		update_page_reclaim_stat(lruvec, file, 0);
+		update_page_reclaim_stat(lruvec, 1, 0);
+		count_vm_events(PGLAZYFREE, hpage_nr_pages(page));
 	}
 }
 

@@ -604,9 +610,9 @@ void lru_add_drain_cpu(int cpu)
 	if (pagevec_count(pvec))
 		pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
 
-	pvec = &per_cpu(lru_deactivate_pvecs, cpu);
+	pvec = &per_cpu(lru_lazyfree_pvecs, cpu);
 	if (pagevec_count(pvec))
-		pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
+		pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
 
 	activate_page_drain(cpu);
 }

@@ -638,22 +644,22 @@ void deactivate_file_page(struct page *page)
 }
 
 /**
- * deactivate_page - deactivate a page
+ * mark_page_lazyfree - make an anon page lazyfree
  * @page: page to deactivate
  *
- * deactivate_page() moves @page to the inactive list if @page was on the active
- * list and was not an unevictable page.  This is done to accelerate the reclaim
- * of @page.
+ * mark_page_lazyfree() moves @page to the inactive file list.
+ * This is done to accelerate the reclaim of @page.
  */
-void deactivate_page(struct page *page)
-{
-	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
-		struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
+void mark_page_lazyfree(struct page *page)
+ {
+	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	    !PageUnevictable(page)) {
+		struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs);
 
 		get_page(page);
 		if (!pagevec_add(pvec, page) || PageCompound(page))
-			pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
-		put_cpu_var(lru_deactivate_pvecs);
+			pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
+		put_cpu_var(lru_lazyfree_pvecs);
 	}
 }
 

@@ -704,7 +710,7 @@ void lru_add_drain_all(void)
 		if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
 		    pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
 		    pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
-		    pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
+		    pagevec_count(&per_cpu(lru_lazyfree_pvecs, cpu)) ||
 		    need_activate_page_drain(cpu)) {
 			INIT_WORK(work, lru_add_drain_per_cpu);
 			queue_work_on(cpu, lru_add_drain_wq, work);

mm/vmstat.c +1 -0

index 69f9aff..7774196 100644
@@ -992,6 +992,7 @@ const char * const vmstat_text[] = {
 	"pgfree",
 	"pgactivate",
 	"pgdeactivate",
+	"pglazyfree",
 
 	"pgfault",
 	"pgmajfault",

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Hi Shaohua,

Quoted 17 lines

Userspace indicates MADV_FREE pages could be freed without pageout, so it pretty much likes used once file pages. For such pages, we'd like to reclaim them once there is memory pressure. Also it might be unfair reclaiming MADV_FREE pages always before used once file pages and we definitively want to reclaim the pages before other anonymous and file pages.

To speed up MADV_FREE pages reclaim, we put the pages into LRU_INACTIVE_FILE list. The rationale is LRU_INACTIVE_FILE list is tiny nowadays and should be full of used once file pages. Reclaiming MADV_FREE pages will not have much interfere of anonymous and active file pages. And the inactive file pages and MADV_FREE pages will be reclaimed according to their age, so we don't reclaim too many MADV_FREE pages too. Putting the MADV_FREE pages into LRU_INACTIVE_FILE_LIST also means we can reclaim the pages without swap support. This idea is suggested by Johannes.

Quoted 2 lines

We also clear the pages SwapBacked flag to indicate they are MADV_FREE pages.

I think this patch should be merged with 3/7. Otherwise, MADV_FREE will be broken during the bisect.

Quoted 23 lines

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 include/linux/mm_inline.h     |  5 +++++
 include/linux/swap.h          |  2 +-
 include/linux/vm_event_item.h |  2 +-
 mm/huge_memory.c              |  5 ++---
 mm/madvise.c                  |  3 +--
 mm/swap.c                     | 50 ++++++++++++++++++++++++-------------------
 mm/vmstat.c                   |  1 +
 7 files changed, 39 insertions(+), 29 deletions(-)

include/linux/mm_inline.h +0 -0

index e030a68..fdded06 100644

include/linux/mm_inline.h static inline int page_is_file_cache(struct page *page) · line 29

Quoted 8 lines

include/linux/mm_inline.h +5 -0

@@ -22,6 +22,11 @@ static inline int page_is_file_cache(struct page *page)
 	return !PageSwapBacked(page);
 }
 
+static inline bool page_is_lazyfree(struct page *page)
+{
+	return PageAnon(page) && !PageSwapBacked(page);
+}
+

trivial:

How about using PageLazyFree for consistency with other PageXXX? As well, use SetPageLazyFree/ClearPageLazyFree rather than using raw {Set,Clear}PageSwapBacked.

include/linux/swap.h extern void lru_add_drain_cpu(int cpu); · line 282

Quoted 13 lines

include/linux/mm_inline.h +0 -0

 static __always_inline void __update_lru_size(struct lruvec *lruvec,
 				enum lru_list lru, enum zone_type zid,
 				int nr_pages)

include/linux/swap.h +1 -1

index 45e91dd..486494e 100644
@@ -279,7 +279,7 @@ extern void lru_add_drain_cpu(int cpu);
 extern void lru_add_drain_all(void);
 extern void rotate_reclaimable_page(struct page *page);
 extern void deactivate_file_page(struct page *page);
-extern void deactivate_page(struct page *page);
+extern void mark_page_lazyfree(struct page *page);

trivial:

How about "deactivate_lazyfree_page"? IMO, it would show intention clear that move the lazy free page to inactive list.

It's just matter of preference so I'm not strong against.

Quoted 71 lines

include/linux/swap.h +0 -0

 extern void swap_setup(void);
 
 extern void add_page_to_unevictable_list(struct page *page);

include/linux/vm_event_item.h +1 -1

index 6aa1b6c..94e58da 100644
@@ -25,7 +25,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		FOR_ALL_ZONES(PGALLOC),
 		FOR_ALL_ZONES(ALLOCSTALL),
 		FOR_ALL_ZONES(PGSCAN_SKIP),
-		PGFREE, PGACTIVATE, PGDEACTIVATE,
+		PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
 		PGFAULT, PGMAJFAULT,
 		PGLAZYFREED,
 		PGREFILL,

mm/huge_memory.c +2 -3

index ecf569d..ddb9a94 100644
@@ -1391,9 +1391,6 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		ClearPageDirty(page);
 	unlock_page(page);
 
-	if (PageActive(page))
-		deactivate_page(page);
-
 	if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
 		orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
 			tlb->fullmm);

@@ -1404,6 +1401,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		set_pmd_at(mm, addr, pmd, orig_pmd);
 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
 	}
+
+	mark_page_lazyfree(page);
 	ret = true;
 out:
 	spin_unlock(ptl);

mm/madvise.c +1 -2

index c867d88..c24549e 100644
@@ -378,10 +378,9 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 			ptent = pte_mkclean(ptent);
 			ptent = pte_wrprotect(ptent);
 			set_pte_at(mm, addr, pte, ptent);
-			if (PageActive(page))
-				deactivate_page(page);
 			tlb_remove_tlb_entry(tlb, pte, addr);
 		}
+		mark_page_lazyfree(page);
 	}
 out:
 	if (nr_swap) {

mm/swap.c +1 -1

index c4910f1..69a7e9d 100644
@@ -46,7 +46,7 @@ int page_cluster;
 static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
-static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
+static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs);
 #ifdef CONFIG_SMP
 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
 #endif

@@ -268,6 +268,11 @@ static void __activate_page(struct page *page, struct lruvec *lruvec,
 		int lru = page_lru_base_type(page);
 

mm/swap.c static void __activate_page(struct page *page, struct lruvec *lruvec, · line 273

Quoted 4 lines

mm/swap.c +3 -0

 		del_page_from_lru_list(page, lruvec, lru);
+		if (page_is_lazyfree(page)) {
+			SetPageSwapBacked(page);
+			file = 0;

I don't see why you set file with 0. Could you explain the rationale?

Quoted 11 lines

include/linux/mm_inline.h static inline int page_is_file_cache(struct page *page) · line 29

Quoted 5 lines

include/linux/mm_inline.h +5 -0

+static inline bool page_is_lazyfree(struct page *page)
+{
+	return PageAnon(page) && !PageSwapBacked(page);
+}
+

trivial:

How about using PageLazyFree for consistency with other PageXXX? As well, use SetPageLazyFree/ClearPageLazyFree rather than using raw {Set,Clear}PageSwapBacked.

So SetPageLazyFree == ClearPageSwapBacked, that would be weird. I personally prefer directly using {Set, Clear}PageSwapBacked, because reader can immediately know what's happening. If using the PageLazyFree, people always need to refer the code and check the relationship between PageLazyFree and PageSwapBacked.

Quoted 17 lines

Quoted 9 lines

static __always_inline void __update_lru_size(struct lruvec *lruvec,

include/linux/mm_inline.h +0 -0

 				enum lru_list lru, enum zone_type zid,
 				int nr_pages)

include/linux/swap.h +0 -0

index 45e91dd..486494e 100644

include/linux/swap.h extern void lru_add_drain_cpu(int cpu); · line 282

Quoted 5 lines

include/linux/swap.h +1 -1

@@ -279,7 +279,7 @@ extern void lru_add_drain_cpu(int cpu);
 extern void lru_add_drain_all(void);
 extern void rotate_reclaimable_page(struct page *page);
 extern void deactivate_file_page(struct page *page);
-extern void deactivate_page(struct page *page);
+extern void mark_page_lazyfree(struct page *page);

trivial:

Quoted 4 lines

How about "deactivate_lazyfree_page"? IMO, it would show intention clear that move the lazy free page to inactive list.

It's just matter of preference so I'm not strong against.

Yes, I thought about the name a little bit. Don't think we should use deactivate, because it sounds that only works for active page, while the function works for both active/inactive pages. I'm open to any suggestions.

Userspace indicates MADV_FREE pages could be freed without pageout, so it pretty much likes used once file pages. For such pages, we'd like to reclaim them once there is memory pressure. Also it might be unfair reclaiming MADV_FREE pages always before used once file pages and we definitively want to reclaim the pages before other anonymous and file pages.

To speed up MADV_FREE pages reclaim, we put the pages into LRU_INACTIVE_FILE list. The rationale is LRU_INACTIVE_FILE list is tiny nowadays and should be full of used once file pages. Reclaiming MADV_FREE pages will not have much interfere of anonymous and active file pages. And the inactive file pages and MADV_FREE pages will be reclaimed according to their age, so we don't reclaim too many MADV_FREE pages too. Putting the MADV_FREE pages into LRU_INACTIVE_FILE_LIST also means we can reclaim the pages without swap support. This idea is suggested by Johannes.

We also clear the pages SwapBacked flag to indicate they are MADV_FREE pages.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 include/linux/mm_inline.h     |  5 +++++
 include/linux/swap.h          |  2 +-
 include/linux/vm_event_item.h |  2 +-
 mm/huge_memory.c              |  5 ++---
 mm/madvise.c                  |  3 +--
 mm/swap.c                     | 50 ++++++++++++++++++++++++-------------------
 mm/vmstat.c                   |  1 +
 7 files changed, 39 insertions(+), 29 deletions(-)

include/linux/mm_inline.h +5 -0

index e030a68..fdded06 100644
@@ -22,6 +22,11 @@ static inline int page_is_file_cache(struct page *page)
 	return !PageSwapBacked(page);
 }
 
+static inline bool page_is_lazyfree(struct page *page)
+{
+	return PageAnon(page) && !PageSwapBacked(page);
+}
+
 static __always_inline void __update_lru_size(struct lruvec *lruvec,
 				enum lru_list lru, enum zone_type zid,
 				int nr_pages)

include/linux/swap.h +1 -1

index 45e91dd..486494e 100644
@@ -279,7 +279,7 @@ extern void lru_add_drain_cpu(int cpu);
 extern void lru_add_drain_all(void);
 extern void rotate_reclaimable_page(struct page *page);
 extern void deactivate_file_page(struct page *page);
-extern void deactivate_page(struct page *page);
+extern void mark_page_lazyfree(struct page *page);
 extern void swap_setup(void);
 
 extern void add_page_to_unevictable_list(struct page *page);

include/linux/vm_event_item.h +1 -1

index 6aa1b6c..94e58da 100644
@@ -25,7 +25,7 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		FOR_ALL_ZONES(PGALLOC),
 		FOR_ALL_ZONES(ALLOCSTALL),
 		FOR_ALL_ZONES(PGSCAN_SKIP),
-		PGFREE, PGACTIVATE, PGDEACTIVATE,
+		PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE,
 		PGFAULT, PGMAJFAULT,
 		PGLAZYFREED,
 		PGREFILL,

mm/huge_memory.c +2 -3

index ecf569d..ddb9a94 100644
@@ -1391,9 +1391,6 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		ClearPageDirty(page);
 	unlock_page(page);
 
-	if (PageActive(page))
-		deactivate_page(page);
-
 	if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
 		orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
 			tlb->fullmm);

@@ -1404,6 +1401,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		set_pmd_at(mm, addr, pmd, orig_pmd);
 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
 	}
+
+	mark_page_lazyfree(page);
 	ret = true;
 out:
 	spin_unlock(ptl);

mm/madvise.c +1 -2

index c867d88..c24549e 100644
@@ -378,10 +378,9 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 			ptent = pte_mkclean(ptent);
 			ptent = pte_wrprotect(ptent);
 			set_pte_at(mm, addr, pte, ptent);
-			if (PageActive(page))
-				deactivate_page(page);
 			tlb_remove_tlb_entry(tlb, pte, addr);
 		}
+		mark_page_lazyfree(page);
 	}
 out:
 	if (nr_swap) {

mm/swap.c +28 -22

index c4910f1..69a7e9d 100644
@@ -46,7 +46,7 @@ int page_cluster;
 static DEFINE_PER_CPU(struct pagevec, lru_add_pvec);
 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs);
-static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
+static DEFINE_PER_CPU(struct pagevec, lru_lazyfree_pvecs);
 #ifdef CONFIG_SMP
 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
 #endif

@@ -268,6 +268,11 @@ static void __activate_page(struct page *page, struct lruvec *lruvec,
 		int lru = page_lru_base_type(page);
 
 		del_page_from_lru_list(page, lruvec, lru);
+		if (page_is_lazyfree(page)) {
+			SetPageSwapBacked(page);
+			file = 0;
+			lru = LRU_INACTIVE_ANON;
+		}
 		SetPageActive(page);
 		lru += LRU_ACTIVE;
 		add_page_to_lru_list(page, lruvec, lru);

@@ -561,20 +566,21 @@ static void lru_deactivate_file_fn(struct page *page, struct lruvec *lruvec,
 }
 
 
-static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
+static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,
 			    void *arg)
 {
-	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
-		int file = page_is_file_cache(page);
-		int lru = page_lru_base_type(page);
+	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	    !PageUnevictable(page)) {
+		bool active = PageActive(page);
 
-		del_page_from_lru_list(page, lruvec, lru + LRU_ACTIVE);
+		del_page_from_lru_list(page, lruvec, LRU_INACTIVE_ANON + active);
 		ClearPageActive(page);
 		ClearPageReferenced(page);
-		add_page_to_lru_list(page, lruvec, lru);
+		ClearPageSwapBacked(page);
+		add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);
 
-		__count_vm_event(PGDEACTIVATE);
-		update_page_reclaim_stat(lruvec, file, 0);
+		update_page_reclaim_stat(lruvec, 1, 0);
+		count_vm_events(PGLAZYFREE, hpage_nr_pages(page));
 	}
 }
 

@@ -604,9 +610,9 @@ void lru_add_drain_cpu(int cpu)
 	if (pagevec_count(pvec))
 		pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL);
 
-	pvec = &per_cpu(lru_deactivate_pvecs, cpu);
+	pvec = &per_cpu(lru_lazyfree_pvecs, cpu);
 	if (pagevec_count(pvec))
-		pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
+		pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
 
 	activate_page_drain(cpu);
 }

@@ -638,22 +644,22 @@ void deactivate_file_page(struct page *page)
 }
 
 /**
- * deactivate_page - deactivate a page
+ * mark_page_lazyfree - make an anon page lazyfree
  * @page: page to deactivate
  *
- * deactivate_page() moves @page to the inactive list if @page was on the active
- * list and was not an unevictable page.  This is done to accelerate the reclaim
- * of @page.
+ * mark_page_lazyfree() moves @page to the inactive file list.
+ * This is done to accelerate the reclaim of @page.
  */
-void deactivate_page(struct page *page)
-{
-	if (PageLRU(page) && PageActive(page) && !PageUnevictable(page)) {
-		struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
+void mark_page_lazyfree(struct page *page)
+ {
+	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	    !PageUnevictable(page)) {
+		struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs);
 
 		get_page(page);
 		if (!pagevec_add(pvec, page) || PageCompound(page))
-			pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
-		put_cpu_var(lru_deactivate_pvecs);
+			pagevec_lru_move_fn(pvec, lru_lazyfree_fn, NULL);
+		put_cpu_var(lru_lazyfree_pvecs);
 	}
 }
 

@@ -704,7 +710,7 @@ void lru_add_drain_all(void)
 		if (pagevec_count(&per_cpu(lru_add_pvec, cpu)) ||
 		    pagevec_count(&per_cpu(lru_rotate_pvecs, cpu)) ||
 		    pagevec_count(&per_cpu(lru_deactivate_file_pvecs, cpu)) ||
-		    pagevec_count(&per_cpu(lru_deactivate_pvecs, cpu)) ||
+		    pagevec_count(&per_cpu(lru_lazyfree_pvecs, cpu)) ||
 		    need_activate_page_drain(cpu)) {
 			INIT_WORK(work, lru_add_drain_per_cpu);
 			queue_work_on(cpu, lru_add_drain_wq, work);

mm/vmstat.c +1 -0

index 69f9aff..7774196 100644
@@ -992,6 +992,7 @@ const char * const vmstat_text[] = {
 	"pgfree",
 	"pgactivate",
 	"pgdeactivate",
+	"pglazyfree",
 
 	"pgfault",
 	"pgmajfault",

Signature · 1 line

2.9.3

Quoted 17 lines · Shaohua Li

Userspace indicates MADV_FREE pages could be freed without pageout, so it pretty much likes used once file pages. For such pages, we'd like to reclaim them once there is memory pressure. Also it might be unfair reclaiming MADV_FREE pages always before used once file pages and we definitively want to reclaim the pages before other anonymous and file pages.

To speed up MADV_FREE pages reclaim, we put the pages into LRU_INACTIVE_FILE list. The rationale is LRU_INACTIVE_FILE list is tiny nowadays and should be full of used once file pages. Reclaiming MADV_FREE pages will not have much interfere of anonymous and active file pages. And the inactive file pages and MADV_FREE pages will be reclaimed according to their age, so we don't reclaim too many MADV_FREE pages too. Putting the MADV_FREE pages into LRU_INACTIVE_FILE_LIST also means we can reclaim the pages without swap support. This idea is suggested by Johannes.

Quoted 2 lines

We also clear the pages SwapBacked flag to indicate they are MADV_FREE pages.

I like this. I have expected this to be more convoluted but it looks quite straightforward. I didn't get to do a really deep review and add my acked-by but from a quick look there do not seem to be any surprises. I was worried about vmstat accounting. There are some places which isolate page from LRU and account based on the LRU and later use page_is_file_cache to tell which LRU this was. This should work fine, though, because you never touch pages which are off-lru.

That being said I do not see any major issues. There might be some minor things and this will need a lot of testing but it is definitely a move into right direction. I hope to do the deeper review after I get back from vacation (20th Feb).

Quoted 7 lines

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

I guess

Suggested-by Johannes Weiner <[email protected]>

would be appropriate.

When memory pressure is high, we free MADV_FREE pages. If the pages are not dirty in pte, the pages could be freed immediately. Otherwise we can't reclaim them. We put the pages back to anonumous LRU list (by setting SwapBacked flag) and the pages will be reclaimed in normal swapout way.

We use normal page reclaim policy. Since MADV_FREE pages are put into inactive file list, such pages and inactive file pages are reclaimed according to their age. This is expected, because we don't want to reclaim too many MADV_FREE pages before used once pages.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 mm/rmap.c   |  4 ++++
 mm/vmscan.c | 43 +++++++++++++++++++++++++++++++------------
 2 files changed, 35 insertions(+), 12 deletions(-)

mm/rmap.c +0 -0

index c8d6204..5f05926 100644

mm/vmscan.c +1 -1

 			return PAGEREF_ACTIVATE;
 		/*
 		 * All mapped pages start out with page table

@@ -903,7 +903,7 @@ static enum page_references page_check_references(struct page *page,
 
 /* Check if a page is dirty or under writeback */
 static void page_check_dirty_writeback(struct page *page,
-				       bool *dirty, bool *writeback)
+			bool *dirty, bool *writeback, bool lazyfree)
 {
 	struct address_space *mapping;
 

mm/vmscan.c +3 -1

 		*dirty = false;
 		*writeback = false;
 		return;

@@ -971,7 +971,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		int may_enter_fs;
 		enum page_references references = PAGEREF_RECLAIM_CLEAN;
 		bool dirty, writeback;
-		bool lazyfree = false;
+		bool lazyfree;
 		int ret = SWAP_SUCCESS;
 
 		cond_resched();

@@ -986,6 +986,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 
 		sc->nr_scanned++;
 
+		lazyfree = page_is_lazyfree(page);
+
 		if (unlikely(!page_evictable(page)))
 			goto cull_mlocked;
 

mm/vmscan.c +1 -1

 		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||

@@ -1005,7 +1007,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		 * will stall and start writing pages if the tail of the LRU
 		 * is all dirty unqueued pages.
 		 */
-		page_check_dirty_writeback(page, &dirty, &writeback);
+		page_check_dirty_writeback(page, &dirty, &writeback, lazyfree);
 		if (dirty || writeback)
 			nr_dirty++;
 

@@ -1107,6 +1109,14 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			; /* try to reclaim the page below */

mm/vmscan.c +15 -6

 		/*
 		 * Anonymous process memory has backing store?
 		 * Try to allocate it some swap space here.

@@ -1116,7 +1126,6 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 				goto keep_locked;
 			if (!add_to_swap(page, page_list))
 				goto activate_locked;
-			lazyfree = true;
 			may_enter_fs = 1;
 
 			/* Adding to swap updated mapping */

@@ -1128,12 +1137,12 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		}
 
 		VM_BUG_ON_PAGE(PageTransHuge(page), page);
-
+unmap_page:
 		/*
 		 * The page is mapped into the page tables of one or more
 		 * processes. Try to unmap it here.
 		 */
-		if (page_mapped(page) && mapping) {
+		if (page_mapped(page) && (mapping || lazyfree)) {
 			switch (ret = try_to_unmap(page, lazyfree ?
 				(ttu_flags | TTU_BATCH_FLUSH | TTU_LZFREE) :
 				(ttu_flags | TTU_BATCH_FLUSH))) {

@@ -1145,7 +1154,14 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			case SWAP_MLOCK:
 				goto cull_mlocked;
 			case SWAP_LZFREE:
-				goto lazyfree;
+				/* follow __remove_mapping for reference */
+				if (page_ref_freeze(page, 1)) {
+					if (!PageDirty(page))
+						goto lazyfree;
+					else
+						page_ref_unfreeze(page, 1);
+				}
+				goto keep_locked;
 			case SWAP_SUCCESS:
 				; /* try to free the page below */
 			}

@@ -1257,10 +1273,9 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			}
 		}
 
-lazyfree:
 		if (!mapping || !__remove_mapping(mapping, page, true))
 			goto keep_locked;
-
+lazyfree:
 		/*
 		 * At this point, we have no other references and there is
 		 * no way to pick any more up (removed from LRU, removed

@@ -1285,6 +1300,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 cull_mlocked:
 		if (PageSwapCache(page))
 			try_to_free_swap(page);
+		if (lazyfree)
+			SetPageSwapBacked(page);
 		unlock_page(page);
 		list_add(&page->lru, &ret_pages);
 		continue;

@@ -1294,6 +1311,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		if (PageSwapCache(page) && mem_cgroup_swap_full(page))
 			try_to_free_swap(page);
 		VM_BUG_ON_PAGE(PageActive(page), page);
+		if (lazyfree)
+			SetPageSwapBacked(page);
 		SetPageActive(page);
 		pgactivate++;
 keep_locked:

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Quoted 29 lines · Shaohua Li

When memory pressure is high, we free MADV_FREE pages. If the pages are not dirty in pte, the pages could be freed immediately. Otherwise we can't reclaim them. We put the pages back to anonumous LRU list (by setting SwapBacked flag) and the pages will be reclaimed in normal swapout way.

We use normal page reclaim policy. Since MADV_FREE pages are put into inactive file list, such pages and inactive file pages are reclaimed according to their age. This is expected, because we don't want to reclaim too many MADV_FREE pages before used once pages.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 mm/rmap.c   |  4 ++++
 mm/vmscan.c | 43 +++++++++++++++++++++++++++++++------------
 2 files changed, 35 insertions(+), 12 deletions(-)

mm/rmap.c +0 -0

index c8d6204..5f05926 100644

mm/rmap.c static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, · line 1560

Quoted 7 lines

mm/rmap.c +4 -0

@@ -1554,6 +1554,10 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 			dec_mm_counter(mm, MM_ANONPAGES);
 			rp->lazyfreed++;
 			goto discard;
+		} else if (flags & TTU_LZFREE) {
+			set_pte_at(mm, address, pte, pteval);
+			ret = SWAP_FAIL;
+			goto out_unmap;

trivial:

How about this?

if (flags && TTU_LZFREE) {
	if (PageDirty(page)) {
		set_pte_at(XXX);
		ret = SWAP_FAIL;
		goto out_unmap;
	} else {
		dec_mm_counter(mm, MM_ANONPAGES);
		rp->lazyfreed++;
		goto discard;
	}
}

Quoted 10 lines

mm/rmap.c +0 -0

 		}
 
 		if (swap_duplicate(entry) < 0) {

mm/vmscan.c +0 -0

index 947ab6f..b304a84 100644
@@ -864,7 +864,7 @@ static enum page_references page_check_references(struct page *page,
 		return PAGEREF_RECLAIM;
 

mm/vmscan.c static enum page_references page_check_references(struct page *page, · line 867

Quoted 3 lines

mm/vmscan.c +1 -1

 	if (referenced_ptes) {
-		if (PageSwapBacked(page))
+		if (PageSwapBacked(page) || PageAnon(page))

If anyone accesses MADV_FREEed range with load op, not store, why shouldn't we discard that pages?

Quoted 13 lines

mm/vmscan.c +1 -1

 			return PAGEREF_ACTIVATE;
 		/*
 		 * All mapped pages start out with page table

@@ -903,7 +903,7 @@ static enum page_references page_check_references(struct page *page,
 
 /* Check if a page is dirty or under writeback */
 static void page_check_dirty_writeback(struct page *page,
-				       bool *dirty, bool *writeback)
+			bool *dirty, bool *writeback, bool lazyfree)
 {
 	struct address_space *mapping;
 

mm/vmscan.c static void page_check_dirty_writeback(struct page *page, · line 914

Quoted 5 lines

mm/vmscan.c +1 -1

@@ -911,7 +911,7 @@ static void page_check_dirty_writeback(struct page *page,
 	 * Anonymous pages are not handled by flushers and must be written
 	 * from reclaim context. Do not stall reclaim based on them
 	 */
-	if (!page_is_file_cache(page)) {
+	if (!page_is_file_cache(page) || lazyfree) {

tivial:

We can check it with PageLazyFree in here rather than passing lazyfree argument. It's consistent like page_is_file_cache in here.

Quoted 24 lines

mm/vmscan.c +3 -1

 		*dirty = false;
 		*writeback = false;
 		return;

@@ -971,7 +971,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		int may_enter_fs;
 		enum page_references references = PAGEREF_RECLAIM_CLEAN;
 		bool dirty, writeback;
-		bool lazyfree = false;
+		bool lazyfree;
 		int ret = SWAP_SUCCESS;
 
 		cond_resched();

@@ -986,6 +986,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 
 		sc->nr_scanned++;
 
+		lazyfree = page_is_lazyfree(page);
+
 		if (unlikely(!page_evictable(page)))
 			goto cull_mlocked;
 

@@ -993,7 +995,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			goto keep_locked;
 

mm/vmscan.c static unsigned long shrink_page_list(struct list_head *page_list, · line 999

Quoted 4 lines

mm/vmscan.c +1 -1

 		/* Double the slab pressure for mapped and swapcache pages */
-		if (page_mapped(page) || PageSwapCache(page))
+		if ((page_mapped(page) || PageSwapCache(page)) && !lazyfree)
 			sc->nr_scanned++;

In this phase, we cannot know whether lazyfree marked page is discarable or not. If it is freeable and mapped, this logic makes sense. However, if the page is dirty?

Quoted 13 lines

mm/vmscan.c +1 -1

 		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||

@@ -1005,7 +1007,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		 * will stall and start writing pages if the tail of the LRU
 		 * is all dirty unqueued pages.
 		 */
-		page_check_dirty_writeback(page, &dirty, &writeback);
+		page_check_dirty_writeback(page, &dirty, &writeback, lazyfree);
 		if (dirty || writeback)
 			nr_dirty++;
 

@@ -1107,6 +1109,14 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			; /* try to reclaim the page below */

mm/vmscan.c static unsigned long shrink_page_list(struct list_head *page_list, · line 1119

Quoted 10 lines

mm/vmscan.c +8 -0

 		}
 
+		/* lazyfree page could be freed directly */
+		if (lazyfree) {
+			if (unlikely(PageTransHuge(page)) &&
+			    split_huge_page_to_list(page, page_list))
+				goto keep_locked;
+			goto unmap_page;
+		}
+

Maybe, we can remove this hunk. Instead add lazyfree check in here.

		if (PageAnon(page) && !PageSwapCache(page) && !lazyfree) {
			if (!(sc->gfp_mask & __GFP_IO))

Quoted 7 lines

Quoted 7 lines

mm/vmscan.c +0 -0

index 947ab6f..b304a84 100644
@@ -864,7 +864,7 @@ static enum page_references page_check_references(struct page *page,
 		return PAGEREF_RECLAIM;
 

Quoted 6 lines

mm/vmscan.c static enum page_references page_check_references(struct page *page, · line 867

Quoted 3 lines

mm/vmscan.c +1 -1

 	if (referenced_ptes) {
-		if (PageSwapBacked(page))
+		if (PageSwapBacked(page) || PageAnon(page))

If anyone accesses MADV_FREEed range with load op, not store, why shouldn't we discard that pages?

Don't have strong opinion about this, userspace probably shouldn't do this. I'm ok to delete it if you insist.

Quoted 4 lines

Quoted 4 lines

mm/vmscan.c +0 -0

 			return PAGEREF_ACTIVATE;
 		/*
 		 * All mapped pages start out with page table

Quoted 21 lines

Quoted 21 lines

mm/vmscan.c +3 -1

@@ -971,7 +971,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		int may_enter_fs;
 		enum page_references references = PAGEREF_RECLAIM_CLEAN;
 		bool dirty, writeback;
-		bool lazyfree = false;
+		bool lazyfree;
 		int ret = SWAP_SUCCESS;
 
 		cond_resched();

@@ -986,6 +986,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 
 		sc->nr_scanned++;
 
+		lazyfree = page_is_lazyfree(page);
+
 		if (unlikely(!page_evictable(page)))
 			goto cull_mlocked;
 

@@ -993,7 +995,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			goto keep_locked;
 

Quoted 8 lines

mm/vmscan.c static unsigned long shrink_page_list(struct list_head *page_list, · line 999

Quoted 4 lines

mm/vmscan.c +1 -1

 		/* Double the slab pressure for mapped and swapcache pages */
-		if (page_mapped(page) || PageSwapCache(page))
+		if ((page_mapped(page) || PageSwapCache(page)) && !lazyfree)
 			sc->nr_scanned++;

In this phase, we cannot know whether lazyfree marked page is discarable or not. If it is freeable and mapped, this logic makes sense. However, if the page is dirty?

I think this doesn't matter. If the page is dirty, it will go to reclaim in next round and swap out. At that time, we will add nr_scanned there.

When memory pressure is high, we free MADV_FREE pages. If the pages are not dirty in pte, the pages could be freed immediately. Otherwise we can't reclaim them. We put the pages back to anonumous LRU list (by setting SwapBacked flag) and the pages will be reclaimed in normal swapout way.

We use normal page reclaim policy. Since MADV_FREE pages are put into inactive file list, such pages and inactive file pages are reclaimed according to their age. This is expected, because we don't want to reclaim too many MADV_FREE pages before used once pages.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 mm/rmap.c   |  4 ++++
 mm/vmscan.c | 43 +++++++++++++++++++++++++++++++------------
 2 files changed, 35 insertions(+), 12 deletions(-)

mm/rmap.c +4 -0

index c8d6204..5f05926 100644
@@ -1554,6 +1554,10 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 			dec_mm_counter(mm, MM_ANONPAGES);
 			rp->lazyfreed++;
 			goto discard;
+		} else if (flags & TTU_LZFREE) {
+			set_pte_at(mm, address, pte, pteval);
+			ret = SWAP_FAIL;
+			goto out_unmap;
 		}
 
 		if (swap_duplicate(entry) < 0) {

mm/vmscan.c +31 -12

index 947ab6f..b304a84 100644
@@ -864,7 +864,7 @@ static enum page_references page_check_references(struct page *page,
 		return PAGEREF_RECLAIM;
 
 	if (referenced_ptes) {
-		if (PageSwapBacked(page))
+		if (PageSwapBacked(page) || PageAnon(page))
 			return PAGEREF_ACTIVATE;
 		/*
 		 * All mapped pages start out with page table

@@ -903,7 +903,7 @@ static enum page_references page_check_references(struct page *page,
 
 /* Check if a page is dirty or under writeback */
 static void page_check_dirty_writeback(struct page *page,
-				       bool *dirty, bool *writeback)
+			bool *dirty, bool *writeback, bool lazyfree)
 {
 	struct address_space *mapping;
 

@@ -911,7 +911,7 @@ static void page_check_dirty_writeback(struct page *page,
 	 * Anonymous pages are not handled by flushers and must be written
 	 * from reclaim context. Do not stall reclaim based on them
 	 */
-	if (!page_is_file_cache(page)) {
+	if (!page_is_file_cache(page) || lazyfree) {
 		*dirty = false;
 		*writeback = false;
 		return;

@@ -971,7 +971,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		int may_enter_fs;
 		enum page_references references = PAGEREF_RECLAIM_CLEAN;
 		bool dirty, writeback;
-		bool lazyfree = false;
+		bool lazyfree;
 		int ret = SWAP_SUCCESS;
 
 		cond_resched();

@@ -986,6 +986,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 
 		sc->nr_scanned++;
 
+		lazyfree = page_is_lazyfree(page);
+
 		if (unlikely(!page_evictable(page)))
 			goto cull_mlocked;
 

@@ -993,7 +995,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			goto keep_locked;
 
 		/* Double the slab pressure for mapped and swapcache pages */
-		if (page_mapped(page) || PageSwapCache(page))
+		if ((page_mapped(page) || PageSwapCache(page)) && !lazyfree)
 			sc->nr_scanned++;
 
 		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||

@@ -1005,7 +1007,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		 * will stall and start writing pages if the tail of the LRU
 		 * is all dirty unqueued pages.
 		 */
-		page_check_dirty_writeback(page, &dirty, &writeback);
+		page_check_dirty_writeback(page, &dirty, &writeback, lazyfree);
 		if (dirty || writeback)
 			nr_dirty++;
 

@@ -1107,6 +1109,14 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			; /* try to reclaim the page below */
 		}
 
+		/* lazyfree page could be freed directly */
+		if (lazyfree) {
+			if (unlikely(PageTransHuge(page)) &&
+			    split_huge_page_to_list(page, page_list))
+				goto keep_locked;
+			goto unmap_page;
+		}
+
 		/*
 		 * Anonymous process memory has backing store?
 		 * Try to allocate it some swap space here.

@@ -1116,7 +1126,6 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 				goto keep_locked;
 			if (!add_to_swap(page, page_list))
 				goto activate_locked;
-			lazyfree = true;
 			may_enter_fs = 1;
 
 			/* Adding to swap updated mapping */

@@ -1128,12 +1137,12 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		}
 
 		VM_BUG_ON_PAGE(PageTransHuge(page), page);
-
+unmap_page:
 		/*
 		 * The page is mapped into the page tables of one or more
 		 * processes. Try to unmap it here.
 		 */
-		if (page_mapped(page) && mapping) {
+		if (page_mapped(page) && (mapping || lazyfree)) {
 			switch (ret = try_to_unmap(page, lazyfree ?
 				(ttu_flags | TTU_BATCH_FLUSH | TTU_LZFREE) :
 				(ttu_flags | TTU_BATCH_FLUSH))) {

@@ -1145,7 +1154,14 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			case SWAP_MLOCK:
 				goto cull_mlocked;
 			case SWAP_LZFREE:
-				goto lazyfree;
+				/* follow __remove_mapping for reference */
+				if (page_ref_freeze(page, 1)) {
+					if (!PageDirty(page))
+						goto lazyfree;
+					else
+						page_ref_unfreeze(page, 1);
+				}
+				goto keep_locked;
 			case SWAP_SUCCESS:
 				; /* try to free the page below */
 			}

@@ -1257,10 +1273,9 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			}
 		}
 
-lazyfree:
 		if (!mapping || !__remove_mapping(mapping, page, true))
 			goto keep_locked;
-
+lazyfree:
 		/*
 		 * At this point, we have no other references and there is
 		 * no way to pick any more up (removed from LRU, removed

@@ -1285,6 +1300,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 cull_mlocked:
 		if (PageSwapCache(page))
 			try_to_free_swap(page);
+		if (lazyfree)
+			SetPageSwapBacked(page);
 		unlock_page(page);
 		list_add(&page->lru, &ret_pages);
 		continue;

@@ -1294,6 +1311,8 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		if (PageSwapCache(page) && mem_cgroup_swap_full(page))
 			try_to_free_swap(page);
 		VM_BUG_ON_PAGE(PageActive(page), page);
+		if (lazyfree)
+			SetPageSwapBacked(page);
 		SetPageActive(page);
 		pgactivate++;
 keep_locked:

Signature · 1 line

2.9.3

Show MADV_FREE pages info in proc/sysfs files.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 drivers/base/node.c       |  2 ++
 fs/proc/meminfo.c         |  1 +
 include/linux/mm_inline.h | 31 ++++++++++++++++++++++++++++---
 include/linux/mmzone.h    |  2 ++
 mm/page_alloc.c           |  7 +++++--
 mm/vmscan.c               |  9 +++++++--
 mm/vmstat.c               |  2 ++
 7 files changed, 47 insertions(+), 7 deletions(-)

drivers/base/node.c +2 -0

index 5548f96..9138db8 100644
@@ -71,6 +71,7 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       "Node %d Active(file):   %8lu kB\n"
 		       "Node %d Inactive(file): %8lu kB\n"
 		       "Node %d Unevictable:    %8lu kB\n"
+		       "Node %d LazyFree:       %8lu kB\n"
 		       "Node %d Mlocked:        %8lu kB\n",
 		       nid, K(i.totalram),
 		       nid, K(i.freeram),

@@ -84,6 +85,7 @@ static ssize_t node_read_meminfo(struct device *dev,
 		       nid, K(node_page_state(pgdat, NR_ACTIVE_FILE)),
 		       nid, K(node_page_state(pgdat, NR_INACTIVE_FILE)),
 		       nid, K(node_page_state(pgdat, NR_UNEVICTABLE)),
+		       nid, K(node_page_state(pgdat, NR_LAZYFREE)),
 		       nid, K(sum_zone_node_page_state(nid, NR_MLOCK)));
 
 #ifdef CONFIG_HIGHMEM

fs/proc/meminfo.c +1 -0

index 8a42849..b2e7b31 100644
@@ -80,6 +80,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
 	show_val_kb(m, "Active(file):   ", pages[LRU_ACTIVE_FILE]);
 	show_val_kb(m, "Inactive(file): ", pages[LRU_INACTIVE_FILE]);
 	show_val_kb(m, "Unevictable:    ", pages[LRU_UNEVICTABLE]);
+	show_val_kb(m, "LazyFree:       ", global_node_page_state(NR_LAZYFREE));
 	show_val_kb(m, "Mlocked:        ", global_page_state(NR_MLOCK));
 
 #ifdef CONFIG_HIGHMEM

include/linux/mm_inline.h +28 -3

index fdded06..3e496de 100644
@@ -48,25 +48,50 @@ static __always_inline void update_lru_size(struct lruvec *lruvec,
 #endif
 }
 
+static __always_inline void __update_lazyfree_size(struct lruvec *lruvec,
+				enum zone_type zid, int nr_pages)
+{
+	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+
+	__mod_node_page_state(pgdat, NR_LAZYFREE, nr_pages);
+	__mod_zone_page_state(&pgdat->node_zones[zid], NR_ZONE_LAZYFREE,
+				nr_pages);
+}
+
 static __always_inline void add_page_to_lru_list(struct page *page,
 				struct lruvec *lruvec, enum lru_list lru)
 {
-	update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page));
+	enum zone_type zid = page_zonenum(page);
+	int nr_pages = hpage_nr_pages(page);
+
+	if (lru == LRU_INACTIVE_FILE && page_is_lazyfree(page))
+		__update_lazyfree_size(lruvec, zid, nr_pages);
+	update_lru_size(lruvec, lru, zid, nr_pages);
 	list_add(&page->lru, &lruvec->lists[lru]);
 }
 
 static __always_inline void add_page_to_lru_list_tail(struct page *page,
 				struct lruvec *lruvec, enum lru_list lru)
 {
-	update_lru_size(lruvec, lru, page_zonenum(page), hpage_nr_pages(page));
+	enum zone_type zid = page_zonenum(page);
+	int nr_pages = hpage_nr_pages(page);
+
+	if (lru == LRU_INACTIVE_FILE && page_is_lazyfree(page))
+		__update_lazyfree_size(lruvec, zid, nr_pages);
+	update_lru_size(lruvec, lru, zid, nr_pages);
 	list_add_tail(&page->lru, &lruvec->lists[lru]);
 }
 
 static __always_inline void del_page_from_lru_list(struct page *page,
 				struct lruvec *lruvec, enum lru_list lru)
 {
+	enum zone_type zid = page_zonenum(page);
+	int nr_pages = hpage_nr_pages(page);
+
 	list_del(&page->lru);
-	update_lru_size(lruvec, lru, page_zonenum(page), -hpage_nr_pages(page));
+	if (lru == LRU_INACTIVE_FILE && page_is_lazyfree(page))
+		__update_lazyfree_size(lruvec, zid, -nr_pages);
+	update_lru_size(lruvec, lru, zid, -nr_pages);
 }
 
 /**

include/linux/mmzone.h +2 -0

index 338a786a..78985f1 100644
@@ -118,6 +118,7 @@ enum zone_stat_item {
 	NR_ZONE_INACTIVE_FILE,
 	NR_ZONE_ACTIVE_FILE,
 	NR_ZONE_UNEVICTABLE,
+	NR_ZONE_LAZYFREE,
 	NR_ZONE_WRITE_PENDING,	/* Count of dirty, writeback and unstable pages */
 	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
 	NR_SLAB_RECLAIMABLE,

@@ -147,6 +148,7 @@ enum node_stat_item {
 	NR_INACTIVE_FILE,	/*  "     "     "   "       "         */
 	NR_ACTIVE_FILE,		/*  "     "     "   "       "         */
 	NR_UNEVICTABLE,		/*  "     "     "   "       "         */
+	NR_LAZYFREE,		/*  "     "     "   "       "         */
 	NR_ISOLATED_ANON,	/* Temporary isolated pages from anon lru */
 	NR_ISOLATED_FILE,	/* Temporary isolated pages from file lru */
 	NR_PAGES_SCANNED,	/* pages scanned since last reclaim */

mm/page_alloc.c +5 -2

index 11b4cd4..d0ff8c2 100644
@@ -4453,7 +4453,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 		" unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n"
 		" slab_reclaimable:%lu slab_unreclaimable:%lu\n"
 		" mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
-		" free:%lu free_pcp:%lu free_cma:%lu\n",
+		" free:%lu free_pcp:%lu free_cma:%lu lazy_free:%lu\n",
 		global_node_page_state(NR_ACTIVE_ANON),
 		global_node_page_state(NR_INACTIVE_ANON),
 		global_node_page_state(NR_ISOLATED_ANON),

@@ -4472,7 +4472,8 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 		global_page_state(NR_BOUNCE),
 		global_page_state(NR_FREE_PAGES),
 		free_pcp,
-		global_page_state(NR_FREE_CMA_PAGES));
+		global_page_state(NR_FREE_CMA_PAGES),
+		global_node_page_state(NR_LAZYFREE));
 
 	for_each_online_pgdat(pgdat) {
 		if (show_mem_node_skip(filter, pgdat->node_id, nodemask))

@@ -4484,6 +4485,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			" active_file:%lukB"
 			" inactive_file:%lukB"
 			" unevictable:%lukB"
+			" lazy_free:%lukB"
 			" isolated(anon):%lukB"
 			" isolated(file):%lukB"
 			" mapped:%lukB"

@@ -4506,6 +4508,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 			K(node_page_state(pgdat, NR_ACTIVE_FILE)),
 			K(node_page_state(pgdat, NR_INACTIVE_FILE)),
 			K(node_page_state(pgdat, NR_UNEVICTABLE)),
+			K(node_page_state(pgdat, NR_LAZYFREE)),
 			K(node_page_state(pgdat, NR_ISOLATED_ANON)),
 			K(node_page_state(pgdat, NR_ISOLATED_FILE)),
 			K(node_page_state(pgdat, NR_FILE_MAPPED)),

mm/vmscan.c +7 -2

index b304a84..1a98467 100644
@@ -1442,7 +1442,8 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode)
  * be complete before mem_cgroup_update_lru_size due to a santity check.
  */
 static __always_inline void update_lru_sizes(struct lruvec *lruvec,
-			enum lru_list lru, unsigned long *nr_zone_taken)
+			enum lru_list lru, unsigned long *nr_zone_taken,
+			unsigned long *nr_zone_lazyfree)
 {
 	int zid;
 

@@ -1450,6 +1451,7 @@ static __always_inline void update_lru_sizes(struct lruvec *lruvec,
 		if (!nr_zone_taken[zid])
 			continue;
 
+		__update_lazyfree_size(lruvec, zid, -nr_zone_lazyfree[zid]);
 		__update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
 #ifdef CONFIG_MEMCG
 		mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);

@@ -1486,6 +1488,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 	struct list_head *src = &lruvec->lists[lru];
 	unsigned long nr_taken = 0;
 	unsigned long nr_zone_taken[MAX_NR_ZONES] = { 0 };
+	unsigned long nr_zone_lazyfree[MAX_NR_ZONES] = { 0 };
 	unsigned long nr_skipped[MAX_NR_ZONES] = { 0, };
 	unsigned long skipped = 0, total_skipped = 0;
 	unsigned long scan, nr_pages;

@@ -1517,6 +1520,8 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 			nr_pages = hpage_nr_pages(page);
 			nr_taken += nr_pages;
 			nr_zone_taken[page_zonenum(page)] += nr_pages;
+			if (page_is_lazyfree(page))
+				nr_zone_lazyfree[page_zonenum(page)] += nr_pages;
 			list_move(&page->lru, dst);
 			break;
 

@@ -1560,7 +1565,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
 	*nr_scanned = scan + total_skipped;
 	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
 				    scan, skipped, nr_taken, mode, lru);
-	update_lru_sizes(lruvec, lru, nr_zone_taken);
+	update_lru_sizes(lruvec, lru, nr_zone_taken, nr_zone_lazyfree);
 	return nr_taken;
 }
 

mm/vmstat.c +2 -0

index 7774196..a70b52d 100644
@@ -926,6 +926,7 @@ const char * const vmstat_text[] = {
 	"nr_zone_inactive_file",
 	"nr_zone_active_file",
 	"nr_zone_unevictable",
+	"nr_zone_lazyfree",
 	"nr_zone_write_pending",
 	"nr_mlock",
 	"nr_slab_reclaimable",

@@ -952,6 +953,7 @@ const char * const vmstat_text[] = {
 	"nr_inactive_file",
 	"nr_active_file",
 	"nr_unevictable",
+	"nr_lazyfree",
 	"nr_isolated_anon",
 	"nr_isolated_file",
 	"nr_pages_scanned",

Signature · 1 line

2.9.3

Quoted 1 line · Shaohua Li

Show MADV_FREE pages info in proc/sysfs files.

How are we going to use this information? Why it isn't sufficient to watch for lazyfree events? I mean this adds quite some code and it is not clear (at least from the changelog) we we need this information.

Quoted 6 lines · Michal Hocko

Quoted 1 line · Shaohua Li

Show MADV_FREE pages info in proc/sysfs files.

How are we going to use this information? Why it isn't sufficient to watch for lazyfree events? I mean this adds quite some code and it is not clear (at least from the changelog) we we need this information.

It's just like any other meminfo we added to let user know what happens in the system. Users can use the info for monitoring/diagnosing. the lazyfree/lazyfreed events can't reflect the lazyfree page info because 'lazyfree - lazyfreed' doesn't equal current lazyfree pages and the events aren't per-node. I'll add more description in the changelog.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 fs/proc/task_mmu.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

fs/proc/task_mmu.c +7 -1

index ee3efb2..8f2423f 100644
@@ -440,6 +440,7 @@ struct mem_size_stats {
 	unsigned long private_dirty;
 	unsigned long referenced;
 	unsigned long anonymous;
+	unsigned long lazyfree;
 	unsigned long anonymous_thp;
 	unsigned long shmem_thp;
 	unsigned long swap;

@@ -456,8 +457,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
 	int i, nr = compound ? 1 << compound_order(page) : 1;
 	unsigned long size = nr * PAGE_SIZE;
 
-	if (PageAnon(page))
+	if (PageAnon(page)) {
 		mss->anonymous += size;
+		if (!PageSwapBacked(page))
+			mss->lazyfree += size;
+	}
 
 	mss->resident += size;
 	/* Accumulate the size in pages that have been accessed. */

@@ -770,6 +774,7 @@ static int show_smap(struct seq_file *m, void *v, int is_pid)
 		   "Private_Dirty:  %8lu kB\n"
 		   "Referenced:     %8lu kB\n"
 		   "Anonymous:      %8lu kB\n"
+		   "LazyFree:       %8lu kB\n"
 		   "AnonHugePages:  %8lu kB\n"
 		   "ShmemPmdMapped: %8lu kB\n"
 		   "Shared_Hugetlb: %8lu kB\n"

@@ -788,6 +793,7 @@ static int show_smap(struct seq_file *m, void *v, int is_pid)
 		   mss.private_dirty >> 10,
 		   mss.referenced >> 10,
 		   mss.anonymous >> 10,
+		   mss.lazyfree >> 10,
 		   mss.anonymous_thp >> 10,
 		   mss.shmem_thp >> 10,
 		   mss.shared_hugetlb >> 10,

Signature · 1 line

2.9.3

$DESCRIPTION_OF_YOUR_USECASE_GOES_HERE

Moreover Documentation/filesystems/proc.txt should be updated as well.

Other than that, the patch looks good to me.

Quoted 8 lines

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

after the description is added and documentation updated

Acked-by Michal Hocko <[email protected]>

Quoted 40 lines

---
 fs/proc/task_mmu.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

fs/proc/task_mmu.c +6 -1

index ee3efb2..8f2423f 100644
@@ -440,6 +440,7 @@ struct mem_size_stats {
 	unsigned long private_dirty;
 	unsigned long referenced;
 	unsigned long anonymous;
+	unsigned long lazyfree;
 	unsigned long anonymous_thp;
 	unsigned long shmem_thp;
 	unsigned long swap;

@@ -456,8 +457,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
 	int i, nr = compound ? 1 << compound_order(page) : 1;
 	unsigned long size = nr * PAGE_SIZE;
 
-	if (PageAnon(page))
+	if (PageAnon(page)) {
 		mss->anonymous += size;
+		if (!PageSwapBacked(page))
+			mss->lazyfree += size;
+	}
 
 	mss->resident += size;
 	/* Accumulate the size in pages that have been accessed. */

@@ -770,6 +774,7 @@ static int show_smap(struct seq_file *m, void *v, int is_pid)
 		   "Private_Dirty:  %8lu kB\n"
 		   "Referenced:     %8lu kB\n"
 		   "Anonymous:      %8lu kB\n"
+		   "LazyFree:       %8lu kB\n"
 		   "AnonHugePages:  %8lu kB\n"
 		   "ShmemPmdMapped: %8lu kB\n"
 		   "Shared_Hugetlb: %8lu kB\n"

@@ -788,6 +793,7 @@ static int show_smap(struct seq_file *m, void *v, int is_pid)
 		   mss.private_dirty >> 10,
 		   mss.referenced >> 10,

Quoted 8 lines

fs/proc/task_mmu.c +1 -0

 		   mss.anonymous >> 10,
+		   mss.lazyfree >> 10,
 		   mss.anonymous_thp >> 10,
 		   mss.shmem_thp >> 10,
 		   mss.shared_hugetlb >> 10,

Signature · 1 line

2.9.3

Signature · 2 lines

Michal Hocko
SUSE Labs

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 fs/proc/task_mmu.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

fs/proc/task_mmu.c +1 -0

index ee3efb2..8f2423f 100644
@@ -440,6 +440,7 @@ struct mem_size_stats {
 	unsigned long private_dirty;
 	unsigned long referenced;
 	unsigned long anonymous;
+	unsigned long lazyfree;
 	unsigned long anonymous_thp;
 	unsigned long shmem_thp;
 	unsigned long swap;

fs/proc/task_mmu.c +4 -0

+			mss->lazyfree += size;
+	}
 
 	mss->resident += size;
 	/* Accumulate the size in pages that have been accessed. */

@@ -770,6 +774,7 @@ static int show_smap(struct seq_file *m, void *v, int is_pid)
 		   "Private_Dirty:  %8lu kB\n"
 		   "Referenced:     %8lu kB\n"
 		   "Anonymous:      %8lu kB\n"
+		   "LazyFree:       %8lu kB\n"
 		   "AnonHugePages:  %8lu kB\n"
 		   "ShmemPmdMapped: %8lu kB\n"
 		   "Shared_Hugetlb: %8lu kB\n"

@@ -788,6 +793,7 @@ static int show_smap(struct seq_file *m, void *v, int is_pid)
 		   mss.private_dirty >> 10,
 		   mss.referenced >> 10,
 		   mss.anonymous >> 10,
+		   mss.lazyfree >> 10,
 		   mss.anonymous_thp >> 10,
 		   mss.shmem_thp >> 10,
 		   mss.shared_hugetlb >> 10,

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Quoted 28 lines · Shaohua Li

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 fs/proc/task_mmu.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

fs/proc/task_mmu.c +1 -0

index ee3efb2..8f2423f 100644
@@ -440,6 +440,7 @@ struct mem_size_stats {
 	unsigned long private_dirty;
 	unsigned long referenced;
 	unsigned long anonymous;
+	unsigned long lazyfree;
 	unsigned long anonymous_thp;
 	unsigned long shmem_thp;
 	unsigned long swap;

@@ -456,8 +457,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
 	int i, nr = compound ? 1 << compound_order(page) : 1;
 	unsigned long size = nr * PAGE_SIZE;
 

fs/proc/task_mmu.c static void smaps_account(struct mem_size_stats *mss, struct page *page, · line 462

Quoted 4 lines

fs/proc/task_mmu.c +2 -1

-	if (PageAnon(page))
+	if (PageAnon(page)) {
 		mss->anonymous += size;
+		if (!PageSwapBacked(page))

How about this?

		if (!PageSwapBacked(page) && !dirty && !PageDirty(page))

Add a separate RSS for MADV_FREE pages. The pages are charged into MM_ANONPAGES (because they are mapped anon pages) and also charged into the MM_LAZYFREEPAGES. /proc/pid/statm will have an extra field to display the RSS, which userspace can use to determine the RSS excluding MADV_FREE pages.

The basic idea is to increment the RSS in madvise and decrement in unmap or page reclaim. There is one limitation. If a page is shared by two processes, since madvise only has mm cotext of current process, it isn't convenient to charge the RSS for both processes. So we don't charge the RSS if the mapcount isn't 1. On the other hand, fork can make a MADV_FREE page shared by two processes. To make things consistent, we uncharge the RSS from the source mm in fork.

A new flag is added to indicate if a page is accounted into the RSS. We can't use SwapBacked flag to do the determination because we can't guarantee the page has SwapBacked flag cleared in madvise. We are reusing mappedtodisk flag which should not be set for Anon pages.

There are a couple of other places we need to uncharge the RSS, activate_page and mark_page_accessed. activate_page is used by swap, where MADV_FREE pages are already not in lazyfree state before going into swap. mark_page_accessed is mainly used for file pages, but there are several places it's used by anonymous pages. I fixed gup, but not some gpu drivers and kvm. If the drivers use MADV_FREE, we might have inprecise RSS accounting.

Please note, the accounting is never going to be precise. MADV_FREE page could be written by userspace without notification to the kernel. The page can't be reclaimed like other clean lazyfree pages. The page isn't real lazyfree page. But since kernel isn't aware of this, the page is still accounted as lazyfree, thus the accounting could be incorrect.

Quoted 27 lines · Shaohua Li

Add a separate RSS for MADV_FREE pages. The pages are charged into MM_ANONPAGES (because they are mapped anon pages) and also charged into the MM_LAZYFREEPAGES. /proc/pid/statm will have an extra field to display the RSS, which userspace can use to determine the RSS excluding MADV_FREE pages.

The basic idea is to increment the RSS in madvise and decrement in unmap or page reclaim. There is one limitation. If a page is shared by two processes, since madvise only has mm cotext of current process, it isn't convenient to charge the RSS for both processes. So we don't charge the RSS if the mapcount isn't 1. On the other hand, fork can make a MADV_FREE page shared by two processes. To make things consistent, we uncharge the RSS from the source mm in fork.

A new flag is added to indicate if a page is accounted into the RSS. We can't use SwapBacked flag to do the determination because we can't guarantee the page has SwapBacked flag cleared in madvise. We are reusing mappedtodisk flag which should not be set for Anon pages.

There are a couple of other places we need to uncharge the RSS, activate_page and mark_page_accessed. activate_page is used by swap, where MADV_FREE pages are already not in lazyfree state before going into swap. mark_page_accessed is mainly used for file pages, but there are several places it's used by anonymous pages. I fixed gup, but not some gpu drivers and kvm. If the drivers use MADV_FREE, we might have inprecise RSS accounting.

Quoted 5 lines

Please note, the accounting is never going to be precise. MADV_FREE page could be written by userspace without notification to the kernel. The page can't be reclaimed like other clean lazyfree pages. The page isn't real lazyfree page. But since kernel isn't aware of this, the page is still accounted as lazyfree, thus the accounting could be incorrect.

This is all quite complex and as you say unprecise already. From the description it is not even clear why do we need it at all. Why is /proc/<pid>/smaps insufficient? I am also not fun of a new page flag - even though you managed to recycle an existing one which is a plus.

Quoted 34 lines · Michal Hocko

Quoted 32 lines · Shaohua Li

Add a separate RSS for MADV_FREE pages. The pages are charged into MM_ANONPAGES (because they are mapped anon pages) and also charged into the MM_LAZYFREEPAGES. /proc/pid/statm will have an extra field to display the RSS, which userspace can use to determine the RSS excluding MADV_FREE pages.

The basic idea is to increment the RSS in madvise and decrement in unmap or page reclaim. There is one limitation. If a page is shared by two processes, since madvise only has mm cotext of current process, it isn't convenient to charge the RSS for both processes. So we don't charge the RSS if the mapcount isn't 1. On the other hand, fork can make a MADV_FREE page shared by two processes. To make things consistent, we uncharge the RSS from the source mm in fork.

A new flag is added to indicate if a page is accounted into the RSS. We can't use SwapBacked flag to do the determination because we can't guarantee the page has SwapBacked flag cleared in madvise. We are reusing mappedtodisk flag which should not be set for Anon pages.

There are a couple of other places we need to uncharge the RSS, activate_page and mark_page_accessed. activate_page is used by swap, where MADV_FREE pages are already not in lazyfree state before going into swap. mark_page_accessed is mainly used for file pages, but there are several places it's used by anonymous pages. I fixed gup, but not some gpu drivers and kvm. If the drivers use MADV_FREE, we might have inprecise RSS accounting.

Please note, the accounting is never going to be precise. MADV_FREE page could be written by userspace without notification to the kernel. The page can't be reclaimed like other clean lazyfree pages. The page isn't real lazyfree page. But since kernel isn't aware of this, the page is still accounted as lazyfree, thus the accounting could be incorrect.

Quoted 4 lines

This is all quite complex and as you say unprecise already. From the description it is not even clear why do we need it at all. Why is /proc/<pid>/smaps insufficient? I am also not fun of a new page flag - even though you managed to recycle an existing one which is a plus.

We have monitor app running in the system to check other apps' RSS and kill them if RSS is abnormal. Checking /proc/pid/smaps is too complicated and slow, don't think we can go that way.

Add a separate RSS for MADV_FREE pages. The pages are charged into MM_ANONPAGES (because they are mapped anon pages) and also charged into the MM_LAZYFREEPAGES. /proc/pid/statm will have an extra field to display the RSS, which userspace can use to determine the RSS excluding MADV_FREE pages.

The basic idea is to increment the RSS in madvise and decrement in unmap or page reclaim. There is one limitation. If a page is shared by two processes, since madvise only has mm cotext of current process, it isn't convenient to charge the RSS for both processes. So we don't charge the RSS if the mapcount isn't 1. On the other hand, fork can make a MADV_FREE page shared by two processes. To make things consistent, we uncharge the RSS from the source mm in fork.

A new flag is added to indicate if a page is accounted into the RSS. We can't use SwapBacked flag to do the determination because we can't guarantee the page has SwapBacked flag cleared in madvise. We are reusing mappedtodisk flag which should not be set for Anon pages.

There are a couple of other places we need to uncharge the RSS, activate_page and mark_page_accessed. activate_page is used by swap, where MADV_FREE pages are already not in lazyfree state before going into swap. mark_page_accessed is mainly used for file pages, but there are several places it's used by anonymous pages. I fixed gup, but not some gpu drivers and kvm. If the drivers use MADV_FREE, we might have inprecise RSS accounting.

Please note, the accounting is never going to be precise. MADV_FREE page could be written by userspace without notification to the kernel. The page can't be reclaimed like other clean lazyfree pages. The page isn't real lazyfree page. But since kernel isn't aware of this, the page is still accounted as lazyfree, thus the accounting could be incorrect.

Cc Michal Hocko <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Hugh Dickins <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Rik van Riel <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Andrew Morton <[email protected]>

Signed-off-by Shaohua Li <[email protected]>

---
 fs/proc/array.c            |  9 ++++++---
 fs/proc/internal.h         |  3 ++-
 fs/proc/task_mmu.c         |  9 +++++++--
 fs/proc/task_nommu.c       |  4 +++-
 include/linux/mm_types.h   |  1 +
 include/linux/page-flags.h |  6 ++++++
 mm/gup.c                   |  2 ++
 mm/huge_memory.c           |  8 ++++++++
 mm/khugepaged.c            |  2 ++
 mm/madvise.c               |  5 +++++
 mm/memory.c                | 13 +++++++++++--
 mm/migrate.c               |  2 ++
 mm/oom_kill.c              | 10 ++++++----
 mm/rmap.c                  |  3 +++
 14 files changed, 64 insertions(+), 13 deletions(-)

fs/proc/array.c +6 -3

index 51a4213..c2281f4 100644
@@ -583,17 +583,19 @@ int proc_pid_statm(struct seq_file *m, struct pid_namespace *ns,
 			struct pid *pid, struct task_struct *task)
 {
 	unsigned long size = 0, resident = 0, shared = 0, text = 0, data = 0;
+	unsigned long lazyfree = 0;
 	struct mm_struct *mm = get_task_mm(task);
 
 	if (mm) {
-		size = task_statm(mm, &shared, &text, &data, &resident);
+		size = task_statm(mm, &shared, &text, &data, &resident,
+				  &lazyfree);
 		mmput(mm);
 	}
 	/*
 	 * For quick read, open code by putting numbers directly
 	 * expected format is
-	 * seq_printf(m, "%lu %lu %lu %lu 0 %lu 0\n",
-	 *               size, resident, shared, text, data);
+	 * seq_printf(m, "%lu %lu %lu %lu 0 %lu 0 %lu\n",
+	 *               size, resident, shared, text, data, lazyfree);
 	 */
 	seq_put_decimal_ull(m, "", size);
 	seq_put_decimal_ull(m, " ", resident);

@@ -602,6 +604,7 @@ int proc_pid_statm(struct seq_file *m, struct pid_namespace *ns,
 	seq_put_decimal_ull(m, " ", 0);
 	seq_put_decimal_ull(m, " ", data);
 	seq_put_decimal_ull(m, " ", 0);
+	seq_put_decimal_ull(m, " ", lazyfree);
 	seq_putc(m, '\n');
 
 	return 0;

fs/proc/internal.h +2 -1

index e2c3c46..6587b9c 100644
@@ -290,5 +290,6 @@ extern const struct file_operations proc_pagemap_operations;
 extern unsigned long task_vsize(struct mm_struct *);
 extern unsigned long task_statm(struct mm_struct *,
 				unsigned long *, unsigned long *,
-				unsigned long *, unsigned long *);
+				unsigned long *, unsigned long *,
+				unsigned long *);
 extern void task_mem(struct seq_file *, struct mm_struct *);

fs/proc/task_mmu.c +7 -2

index 8f2423f..f18b568 100644
@@ -23,9 +23,10 @@
 
 void task_mem(struct seq_file *m, struct mm_struct *mm)
 {
-	unsigned long text, lib, swap, ptes, pmds, anon, file, shmem;
+	unsigned long text, lib, swap, ptes, pmds, anon, file, shmem, lazyfree;
 	unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
 
+	lazyfree = get_mm_counter(mm, MM_LAZYFREEPAGES);
 	anon = get_mm_counter(mm, MM_ANONPAGES);
 	file = get_mm_counter(mm, MM_FILEPAGES);
 	shmem = get_mm_counter(mm, MM_SHMEMPAGES);

@@ -59,6 +60,7 @@ void task_mem(struct seq_file *m, struct mm_struct *mm)
 		"RssAnon:\t%8lu kB\n"
 		"RssFile:\t%8lu kB\n"
 		"RssShmem:\t%8lu kB\n"
+		"RssLazyfree:\t%8lu kB\n"
 		"VmData:\t%8lu kB\n"
 		"VmStk:\t%8lu kB\n"
 		"VmExe:\t%8lu kB\n"

@@ -75,6 +77,7 @@ void task_mem(struct seq_file *m, struct mm_struct *mm)
 		anon << (PAGE_SHIFT-10),
 		file << (PAGE_SHIFT-10),
 		shmem << (PAGE_SHIFT-10),
+		lazyfree << (PAGE_SHIFT-10),
 		mm->data_vm << (PAGE_SHIFT-10),
 		mm->stack_vm << (PAGE_SHIFT-10), text, lib,
 		ptes >> 10,

@@ -90,7 +93,8 @@ unsigned long task_vsize(struct mm_struct *mm)
 
 unsigned long task_statm(struct mm_struct *mm,
 			 unsigned long *shared, unsigned long *text,
-			 unsigned long *data, unsigned long *resident)
+			 unsigned long *data, unsigned long *resident,
+			 unsigned long *lazyfree)
 {
 	*shared = get_mm_counter(mm, MM_FILEPAGES) +
 			get_mm_counter(mm, MM_SHMEMPAGES);

@@ -98,6 +102,7 @@ unsigned long task_statm(struct mm_struct *mm,
 								>> PAGE_SHIFT;
 	*data = mm->data_vm + mm->stack_vm;
 	*resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
+	*lazyfree = get_mm_counter(mm, MM_LAZYFREEPAGES);
 	return mm->total_vm;
 }
 

fs/proc/task_nommu.c +3 -1

index 1ef97cf..50426de 100644
@@ -94,7 +94,8 @@ unsigned long task_vsize(struct mm_struct *mm)
 
 unsigned long task_statm(struct mm_struct *mm,
 			 unsigned long *shared, unsigned long *text,
-			 unsigned long *data, unsigned long *resident)
+			 unsigned long *data, unsigned long *resident,
+			 unsigned long *lazyfree)
 {
 	struct vm_area_struct *vma;
 	struct vm_region *region;

@@ -120,6 +121,7 @@ unsigned long task_statm(struct mm_struct *mm,
 	size >>= PAGE_SHIFT;
 	size += *text + *data;
 	*resident = size;
+	*lazyfree = 0;
 	return size;
 }
 

include/linux/mm_types.h +1 -0

index 4f6d440..b6a1428 100644
@@ -376,6 +376,7 @@ enum {
 	MM_ANONPAGES,	/* Resident anonymous pages */
 	MM_SWAPENTS,	/* Anonymous swap entries */
 	MM_SHMEMPAGES,	/* Resident shared memory pages */
+	MM_LAZYFREEPAGES, /* Lazyfree pages, also charged into MM_ANONPAGES */
 	NR_MM_COUNTERS
 };
 

include/linux/page-flags.h +6 -0

index 6b5818d..67c732b 100644
@@ -107,6 +107,8 @@ enum pageflags {
 #endif
 	__NR_PAGEFLAGS,
 
+	PG_lazyfreeaccounted = PG_mappedtodisk, /* only for anon MADV_FREE pages */
+
 	/* Filesystems */
 	PG_checked = PG_owner_priv_1,
 

@@ -428,6 +430,10 @@ TESTPAGEFLAG_FALSE(Ksm)
 
 u64 stable_page_flags(struct page *page);
 
+PAGEFLAG(LazyFreeAccounted, lazyfreeaccounted, PF_ANY)
+	TESTSETFLAG(LazyFreeAccounted, lazyfreeaccounted, PF_ANY)
+	TESTCLEARFLAG(LazyFreeAccounted, lazyfreeaccounted, PF_ANY)
+
 static inline int PageUptodate(struct page *page)
 {
 	int ret;

mm/gup.c +2 -0

index 40abe4c..e64d990 100644
@@ -171,6 +171,8 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
 		 * mark_page_accessed().
 		 */
 		mark_page_accessed(page);
+		if (PageAnon(page) && TestClearPageLazyFreeAccounted(page))
+			dec_mm_counter(mm, MM_LAZYFREEPAGES);
 	}
 	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
 		/* Do not mlock pte-mapped THP */

mm/huge_memory.c +8 -0

index ddb9a94..951fa34 100644
@@ -871,6 +871,8 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
 	get_page(src_page);
 	page_dup_rmap(src_page, true);
+	if (PageAnon(src_page) && TestClearPageLazyFreeAccounted(src_page))
+		add_mm_counter(src_mm, MM_LAZYFREEPAGES, -HPAGE_PMD_NR);
 	add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
 	atomic_long_inc(&dst_mm->nr_ptes);
 	pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);

@@ -1402,6 +1404,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
 	}
 
+	if (page_mapcount(page) == 1 && !TestSetPageLazyFreeAccounted(page))
+		add_mm_counter(mm, MM_LAZYFREEPAGES, HPAGE_PMD_NR);
 	mark_page_lazyfree(page);
 	ret = true;
 out:

@@ -1459,6 +1463,9 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			pte_free(tlb->mm, pgtable);
 			atomic_long_dec(&tlb->mm->nr_ptes);
 			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
+			if (TestClearPageLazyFreeAccounted(page))
+				add_mm_counter(tlb->mm, MM_LAZYFREEPAGES,
+						-HPAGE_PMD_NR);
 		} else {
 			if (arch_needs_pgtable_deposit())
 				zap_deposited_table(tlb->mm, pmd);

@@ -1917,6 +1924,7 @@ static void __split_huge_page_tail(struct page *head, int tail,
 			 (1L << PG_swapbacked) |
 			 (1L << PG_mlocked) |
 			 (1L << PG_uptodate) |
+			 (1L << PG_lazyfreeaccounted) |
 			 (1L << PG_active) |
 			 (1L << PG_locked) |
 			 (1L << PG_unevictable) |

mm/khugepaged.c +2 -0

index a4b499f..e4668db 100644
@@ -577,6 +577,8 @@ static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
 		}
 		inc_node_page_state(page,
 				NR_ISOLATED_ANON + page_is_file_cache(page));
+		if (TestClearPageLazyFreeAccounted(page))
+			dec_mm_counter(vma->vm_mm, MM_LAZYFREEPAGES);
 		VM_BUG_ON_PAGE(!PageLocked(page), page);
 		VM_BUG_ON_PAGE(PageLRU(page), page);
 

mm/madvise.c +5 -0

index fe40e93..3c90956 100644
@@ -275,6 +275,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 	struct page *page;
 	int nr_swap = 0;
 	unsigned long next;
+	int nr_lazyfree_accounted = 0;
 
 	next = pmd_addr_end(addr, end);
 	if (pmd_trans_huge(*pmd))

@@ -380,9 +381,13 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
 			set_pte_at(mm, addr, pte, ptent);
 			tlb_remove_tlb_entry(tlb, pte, addr);
 		}
+		if (page_mapcount(page) == 1 &&
+		    !TestSetPageLazyFreeAccounted(page))
+			nr_lazyfree_accounted++;
 		mark_page_lazyfree(page);
 	}
 out:
+	add_mm_counter(mm, MM_LAZYFREEPAGES, nr_lazyfree_accounted);
 	if (nr_swap) {
 		if (current->mm == mm)
 			sync_mm_rss(mm);

mm/memory.c +11 -2

index cf97d88..e275de1 100644
@@ -850,7 +850,7 @@ struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
 static inline unsigned long
 copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 		pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
-		unsigned long addr, int *rss)
+		unsigned long addr, int *rss, int *rss_src_lazyfree)
 {
 	unsigned long vm_flags = vma->vm_flags;
 	pte_t pte = *src_pte;

@@ -915,6 +915,9 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	if (page) {
 		get_page(page);
 		page_dup_rmap(page, false);
+		if (PageAnon(page) &&
+		    TestClearPageLazyFreeAccounted(page))
+			(*rss_src_lazyfree)++;
 		rss[mm_counter(page)]++;
 	}
 

@@ -932,10 +935,12 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 	spinlock_t *src_ptl, *dst_ptl;
 	int progress = 0;
 	int rss[NR_MM_COUNTERS];
+	int rss_src_lazyfree;
 	swp_entry_t entry = (swp_entry_t){0};
 
 again:
 	init_rss_vec(rss);
+	rss_src_lazyfree = 0;
 
 	dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
 	if (!dst_pte)

@@ -963,13 +968,14 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
 			continue;
 		}
 		entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte,
-							vma, addr, rss);
+					vma, addr, rss, &rss_src_lazyfree);
 		if (entry.val)
 			break;
 		progress += 8;
 	} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
 
 	arch_leave_lazy_mmu_mode();
+	add_mm_counter(src_mm, MM_LAZYFREEPAGES, -rss_src_lazyfree);
 	spin_unlock(src_ptl);
 	pte_unmap(orig_src_pte);
 	add_mm_rss_vec(dst_mm, rss);

@@ -1163,6 +1169,9 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
 					mark_page_accessed(page);
 			}
 			rss[mm_counter(page)]--;
+			if (PageAnon(page) &&
+			    TestClearPageLazyFreeAccounted(page))
+				rss[MM_LAZYFREEPAGES]--;
 			page_remove_rmap(page, false);
 			if (unlikely(page_mapcount(page) < 0))
 				print_bad_pte(vma, addr, ptent, page);

mm/migrate.c +2 -0

index eb76f87..6e586d2 100644
@@ -642,6 +642,8 @@ void migrate_page_copy(struct page *newpage, struct page *page)
 		SetPageChecked(newpage);
 	if (PageMappedToDisk(page))
 		SetPageMappedToDisk(newpage);
+	if (PageLazyFreeAccounted(page))
+		SetPageLazyFreeAccounted(newpage);
 
 	/* Move dirty on pages not done by migrate_page_move_mapping() */
 	if (PageDirty(page))

mm/oom_kill.c +6 -4

index 51c0918..54e0604 100644
@@ -528,11 +528,12 @@ static bool __oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm)
 					 NULL);
 	}
 	tlb_finish_mmu(&tlb, 0, -1);
-	pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n",
+	pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, lazyfree-rss:%lukB\n",
 			task_pid_nr(tsk), tsk->comm,
 			K(get_mm_counter(mm, MM_ANONPAGES)),
 			K(get_mm_counter(mm, MM_FILEPAGES)),
-			K(get_mm_counter(mm, MM_SHMEMPAGES)));
+			K(get_mm_counter(mm, MM_SHMEMPAGES)),
+			K(get_mm_counter(mm, MM_LAZYFREEPAGES)));
 	up_read(&mm->mmap_sem);
 
 	/*

@@ -878,11 +879,12 @@ static void oom_kill_process(struct oom_control *oc, const char *message)
 	 */
 	do_send_sig_info(SIGKILL, SEND_SIG_FORCED, victim, true);
 	mark_oom_victim(victim);
-	pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n",
+	pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, lazyfree-rss:%lukB\n",
 		task_pid_nr(victim), victim->comm, K(victim->mm->total_vm),
 		K(get_mm_counter(victim->mm, MM_ANONPAGES)),
 		K(get_mm_counter(victim->mm, MM_FILEPAGES)),
-		K(get_mm_counter(victim->mm, MM_SHMEMPAGES)));
+		K(get_mm_counter(victim->mm, MM_SHMEMPAGES)),
+		K(get_mm_counter(victim->mm, MM_LAZYFREEPAGES)));
 	task_unlock(victim);
 
 	/*

mm/rmap.c +3 -0

index 5f05926..86c80d7 100644
@@ -1585,6 +1585,9 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 	put_page(page);
 
 out_unmap:
+	/* regardless of success or failure, the page isn't lazyfree */
+	if (PageAnon(page) && TestClearPageLazyFreeAccounted(page))
+		add_mm_counter(mm, MM_LAZYFREEPAGES, -hpage_nr_pages(page));
 	pte_unmap_unlock(pte, ptl);
 	if (ret != SWAP_FAIL && ret != SWAP_MLOCK && !(flags & TTU_MUNLOCK))
 		mmu_notifier_invalidate_page(mm, address);

Signature · 7 lines

2.9.3

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>