mm: Add PageLayzyFree() helper functions for MADV_FREE

PATCH In review Huang, Ying 3 participants 4 messages Mar 3, 2020 – Mar 4, 2020

Subsystems (5)

Memory management MM core MGLRU MM reclaim +1 more subsystems

lore

Share & tools

lore

Huang, Ying

author

2020-03-03 03:37 UTC #0 lore raw

PATCH 1/1 mm: Add PageLayzyFree() helper functions for MADV_FREE

From: Huang Ying <[email protected]>

Now PageSwapBacked() is used as the helper function to check whether pages have been freed lazily via MADV_FREE. This isn't very obvious. So Dave suggested to add PageLazyFree() family helper functions to improve the code readability.

Signed-off-by "Huang, Ying" <[email protected]>

Cc David Hildenbrand <[email protected]>

Cc Mel Gorman <[email protected]>

Cc Vlastimil Babka <[email protected]>

Cc Zi Yan <[email protected]>

Cc Michal Hocko <[email protected]>

Cc Peter Zijlstra <[email protected]>

Cc Minchan Kim <[email protected]>

Cc Johannes Weiner <[email protected]>

Cc Hugh Dickins <[email protected]>

---
 include/linux/page-flags.h | 25 +++++++++++++++++++++++++
 mm/rmap.c                  |  6 +++---
 mm/swap.c                  | 11 +++--------
 mm/vmscan.c                |  7 +++----
 4 files changed, 34 insertions(+), 15 deletions(-)

include/linux/page-flags.h +25 -0

index 49c2697046b9..759748fbcfad 100644
@@ -498,6 +498,31 @@ static __always_inline int PageKsm(struct page *page)
 TESTPAGEFLAG_FALSE(Ksm)
 #endif
 
+/*
+ * For pages freed lazily via MADV_FREE.  lazyfree pages are clean
+ * anonymous pages.  They have SwapBacked flag cleared to distinguish
+ * with normal anonymous pages
+ */
+static __always_inline int PageLazyFree(struct page *page)
+{
+	page = compound_head(page);
+	return PageAnon(page) && !PageSwapBacked(page);
+}
+
+static __always_inline void SetPageLazyFree(struct page *page)
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);
+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	ClearPageSwapBacked(page);
+}
+
+static __always_inline void ClearPageLazyFree(struct page *page)
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);
+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	SetPageSwapBacked(page);
+}
+
 u64 stable_page_flags(struct page *page);
 
 static inline int PageUptodate(struct page *page)

mm/rmap.c +3 -3

index 03c5b116d30e..1dcbb1771dd7 100644
@@ -1598,7 +1598,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 			 * Store the swap location in the pte.
 			 * See handle_pte_fault() ...
 			 */
-			if (unlikely(PageSwapBacked(page) != PageSwapCache(page))) {
+			if (unlikely(PageLazyFree(page) == PageSwapCache(page))) {
 				WARN_ON_ONCE(1);
 				ret = false;
 				/* We have to invalidate as we cleared the pte */

@@ -1609,7 +1609,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 			}
 
 			/* MADV_FREE page check */
-			if (!PageSwapBacked(page)) {
+			if (PageLazyFree(page)) {
 				if (!PageDirty(page)) {
 					/* Invalidate as we cleared the pte */
 					mmu_notifier_invalidate_range(mm,

@@ -1623,7 +1623,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 				 * discarded. Remap the page to page table.
 				 */
 				set_pte_at(mm, address, pvmw.pte, pteval);
-				SetPageSwapBacked(page);
+				ClearPageLazyFree(page);
 				ret = false;
 				page_vma_mapped_walk_done(&pvmw);
 				break;

mm/swap.c +3 -8

index f502a2155e85..bd5e40e14c94 100644
@@ -564,7 +564,7 @@ static void lru_deactivate_fn(struct page *page, struct lruvec *lruvec,
 static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,
 			    void *arg)
 {
-	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	if (PageLRU(page) && PageAnon(page) && !PageLazyFree(page) &&
 	    !PageSwapCache(page) && !PageUnevictable(page)) {
 		bool active = PageActive(page);

@@ -572,12 +572,7 @@ static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec,
 				       LRU_INACTIVE_ANON + active);
 		ClearPageActive(page);
 		ClearPageReferenced(page);
-		/*
-		 * lazyfree pages are clean anonymous pages. They have
-		 * SwapBacked flag cleared to distinguish normal anonymous
-		 * pages
-		 */
-		ClearPageSwapBacked(page);
+		SetPageLazyFree(page);
 		add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);
 
 		__count_vm_events(PGLAZYFREE, hpage_nr_pages(page));

@@ -678,7 +673,7 @@ void deactivate_page(struct page *page)
  */
 void mark_page_lazyfree(struct page *page)
 {
-	if (PageLRU(page) && PageAnon(page) && PageSwapBacked(page) &&
+	if (PageLRU(page) && PageAnon(page) && !PageLazyFree(page) &&
 	    !PageSwapCache(page) && !PageUnevictable(page)) {
 		struct pagevec *pvec = &get_cpu_var(lru_lazyfree_pvecs);

mm/vmscan.c +3 -4

index f14c8c6069a6..0aaee7052fb0 100644
@@ -1043,8 +1043,7 @@ static void page_check_dirty_writeback(struct page *page,
 	 * Anonymous pages are not handled by flushers and must be written
 	 * from reclaim context. Do not stall reclaim based on them
 	 */
-	if (!page_is_file_cache(page) ||
-	    (PageAnon(page) && !PageSwapBacked(page))) {
+	if (!page_is_file_cache(page) || PageLazyFree(page)) {
 		*dirty = false;
 		*writeback = false;
 		return;

@@ -1235,7 +1234,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 		 * Try to allocate it some swap space here.
 		 * Lazyfree page could be freed directly
 		 */
-		if (PageAnon(page) && PageSwapBacked(page)) {
+		if (PageAnon(page) && !PageLazyFree(page)) {
 			if (!PageSwapCache(page)) {
 				if (!(sc->gfp_mask & __GFP_IO))
 					goto keep_locked;

@@ -1411,7 +1410,7 @@ static unsigned long shrink_page_list(struct list_head *page_list,
 			}
 		}
 
-		if (PageAnon(page) && !PageSwapBacked(page)) {
+		if (PageLazyFree(page)) {
 			/* follow __remove_mapping for reference */
 			if (!page_ref_freeze(page, 1))
 				goto keep_locked;

Signature · 1 line

2.25.0

Andrew Morton

maintainer

2020-03-04 00:59 UTC #1 lore raw

Quoted 27 lines · Ying

From: Huang Ying <[email protected]>
Now PageSwapBacked() is used as the helper function to check whether pages have been freed lazily via MADV_FREE. This isn't very obvious. So Dave suggested to add PageLazyFree() family helper functions to improve the code readability.
include/linux/page-flags.h +14 -0
@@ -498,6 +498,31 @@ static __always_inline int PageKsm(struct page *page)
 TESTPAGEFLAG_FALSE(Ksm)
 #endif
 
+/*
+ * For pages freed lazily via MADV_FREE.  lazyfree pages are clean
+ * anonymous pages.  They have SwapBacked flag cleared to distinguish
+ * with normal anonymous pages
+ */
+static __always_inline int PageLazyFree(struct page *page)
+{
+	page = compound_head(page);
+	return PageAnon(page) && !PageSwapBacked(page);
+}
+
+static __always_inline void SetPageLazyFree(struct page *page)
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);

include/linux/page-flags.h static __always_inline int PageKsm(struct page *page) · line 524

Quoted 10 lines

include/linux/page-flags.h +10 -0

+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	ClearPageSwapBacked(page);
+}
+
+static __always_inline void ClearPageLazyFree(struct page *page)
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);
+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	SetPageSwapBacked(page);
+}

These BUG_ONs aren't present in the current code and are unchangelogged.

And they aren't free!

before:

q:/usr/src/25> size -t mm/rmap.o mm/swap.o mm/vmscan.o

   text    data     bss     dec     hex filename
  41580    4211     192   45983    b39f mm/rmap.o
  50684    9259    1184   61127    eec7 mm/swap.o
 117728   46775     128  164631   28317 mm/vmscan.o
 209992   60245    1504  271741   4257d (TOTALS)

after:

   text    data     bss     dec     hex filename
  42035    4299     192   46526    b5be mm/rmap.o
  51091    9347    1184   61622    f0b6 mm/swap.o
 118132   46775     128  165035   284ab mm/vmscan.o
 211258   60421    1504  273183   42b1f (TOTALS)

ALso, if they actually trigger, Linus goes berserk.

If you have some special reason to add these assertions, please do it as a (changelogged!) followup patch and I'll keep it as a linux-next only thing.

Huang, Ying

author

2020-03-04 01:20 UTC #2 lore raw

Hi, Andrew,

Quoted 62 lines · Andrew Morton

Quoted 27 lines · Ying
From: Huang Ying <[email protected]>
Now PageSwapBacked() is used as the helper function to check whether pages have been freed lazily via MADV_FREE. This isn't very obvious. So Dave suggested to add PageLazyFree() family helper functions to improve the code readability.
include/linux/page-flags.h +14 -0
@@ -498,6 +498,31 @@ static __always_inline int PageKsm(struct page *page)
 TESTPAGEFLAG_FALSE(Ksm)
 #endif
 
+/*
+ * For pages freed lazily via MADV_FREE.  lazyfree pages are clean
+ * anonymous pages.  They have SwapBacked flag cleared to distinguish
+ * with normal anonymous pages
+ */
+static __always_inline int PageLazyFree(struct page *page)
+{
+	page = compound_head(page);
+	return PageAnon(page) && !PageSwapBacked(page);
+}
+
+static __always_inline void SetPageLazyFree(struct page *page)
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);
include/linux/page-flags.h static __always_inline int PageKsm(struct page *page) · line 524
Quoted 10 lines
include/linux/page-flags.h +10 -0
+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	ClearPageSwapBacked(page);
+}
+
+static __always_inline void ClearPageLazyFree(struct page *page)
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);
+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	SetPageSwapBacked(page);
+}
These BUG_ONs aren't present in the current code and are unchangelogged.
And they aren't free!
before:
q:/usr/src/25> size -t mm/rmap.o mm/swap.o mm/vmscan.o
   text    data     bss     dec     hex filename
  41580    4211     192   45983    b39f mm/rmap.o
  50684    9259    1184   61127    eec7 mm/swap.o
 117728   46775     128  164631   28317 mm/vmscan.o
 209992   60245    1504  271741   4257d (TOTALS)
after:
   text    data     bss     dec     hex filename
  42035    4299     192   46526    b5be mm/rmap.o
  51091    9347    1184   61622    f0b6 mm/swap.o
 118132   46775     128  165035   284ab mm/vmscan.o
 211258   60421    1504  273183   42b1f (TOTALS)

Quoted 4 lines

ALso, if they actually trigger, Linus goes berserk.
If you have some special reason to add these assertions, please do it as a (changelogged!) followup patch and I'll keep it as a linux-next only thing.

Sorry, originally I thought that VM_BUG_ON_PAGE() will not be used in the production kernel, so it's cheap. But it seems that people do care about its usage. I will send v2 to fix this.

Best Regards,
Huang, Ying

David Rientjes

2020-03-04 01:27 UTC #3 lore raw

In reply to Andrew Morton

Quoted 34 lines · Andrew Morton

Quoted 32 lines · Ying

From: Huang Ying <[email protected]>

include/linux/page-flags.h +19 -0

@@ -498,6 +498,31 @@ static __always_inline int PageKsm(struct page *page)
 TESTPAGEFLAG_FALSE(Ksm)
 #endif
 
+/*
+ * For pages freed lazily via MADV_FREE.  lazyfree pages are clean
+ * anonymous pages.  They have SwapBacked flag cleared to distinguish
+ * with normal anonymous pages
+ */
+static __always_inline int PageLazyFree(struct page *page)
+{
+	page = compound_head(page);
+	return PageAnon(page) && !PageSwapBacked(page);
+}
+
+static __always_inline void SetPageLazyFree(struct page *page)
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);
+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	ClearPageSwapBacked(page);
+}
+
+static __always_inline void ClearPageLazyFree(struct page *page)

Quoted 9 lines

include/linux/page-flags.h static __always_inline int PageKsm(struct page *page) · line 524
Quoted 5 lines
include/linux/page-flags.h +5 -0
+{
+	VM_BUG_ON_PAGE(PageTail(page), page);
+	VM_BUG_ON_PAGE(!PageAnon(page), page);
+	SetPageSwapBacked(page);
+}
These BUG_ONs aren't present in the current code and are unchangelogged.

Yeah, as well as the implicit conversion to check compound_head().