Skip to contentSkip to conversation

mm/swap: stop using page->private on tail pages for THP_SWAP + cleanups

PATCH In review David Hildenbrand 4 participants 17 messages Aug 21, 2023 – Aug 23, 2023

Subsystems (10)
lore
Share & tools
lore
#0 lore raw

cover mm/swap: stop using page->private on tail pages for THP_SWAP + cleanups

This series stops using page->private on tail pages for THP_SWAP, replaces folio->private by folio->swap for swapcache folios, and starts using "new_folio" for tail pages that we are splitting to remove the usage of page->private for swapcache handling completely.

@Willy, I added your SOB on patch #2 from [1], hoping that's ok.

[1] https://lore.kernel.org/all/ZN1IH/[email protected]/

Cc Andrew Morton <[email protected]>
Cc Matthew Wilcox <[email protected]>
Cc Peter Xu <[email protected]>
Cc Catalin Marinas <[email protected]>
Cc Will Deacon <[email protected]>
Cc Hugh Dickins <[email protected]>
Cc Seth Jennings <[email protected]>
Cc Dan Streetman <[email protected]>
Cc Vitaly Wool <[email protected]>

David Hildenbrand (3):
mm/swap: stop using page->private on tail pages for THP_SWAP mm/swap: inline folio_set_swap_entry() and folio_swap_entry() mm/huge_memory: work on folio->swap instead of page->private when
splitting folio

Matthew Wilcox (1):
mm/swap: use dedicated entry for swap in folio

 arch/arm64/mm/mteswap.c  |  5 +++--
 include/linux/mm_types.h | 35 ++++++++++++++---------------------
 include/linux/swap.h     | 12 +++++-------
 mm/huge_memory.c         | 40 +++++++++++++++++++---------------------
 mm/memory.c              |  4 ++--
 mm/rmap.c                |  2 +-
 mm/shmem.c               |  6 +++---
 mm/swap_state.c          |  6 +++---
 mm/swapfile.c            |  6 +++---
 mm/util.c                |  2 +-
 mm/vmscan.c              |  2 +-
 mm/zswap.c               |  4 ++--
 12 files changed, 57 insertions(+), 67 deletions(-)
Signature · 1 line
2.41.0
#1 lore raw

PATCH 1/4 mm/swap: stop using page->private on tail pages for THP_SWAP

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

---
 arch/arm64/mm/mteswap.c  |  5 +++--
 include/linux/mm_types.h | 12 +-----------
 include/linux/swap.h     |  9 +++++++++
 mm/huge_memory.c         | 15 ++++++---------
 mm/memory.c              |  2 +-
 mm/rmap.c                |  2 +-
 mm/swap_state.c          |  5 +++--
 mm/swapfile.c            |  4 ++--
 8 files changed, 26 insertions(+), 28 deletions(-)
arch/arm64/mm/mteswap.c +3 -2
index cd508ba80ab1..a31833e3ddc5 100644
@@ -33,8 +33,9 @@ int mte_save_tags(struct page *page)
 
 	mte_save_page_tags(page_address(page), tag_storage);
 
-	/* page_private contains the swap entry.val set in do_swap_page */
-	ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
+	/* lookup the swap entry.val from the page */
+	ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage,
+		       GFP_KERNEL);
 	if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
 		mte_free_tag_storage(tag_storage);
 		return xa_err(ret);
include/linux/mm_types.h +1 -11
index b9b6c88875b9..61361f1750c3 100644
@@ -333,11 +333,8 @@ struct folio {
 			atomic_t _pincount;
 #ifdef CONFIG_64BIT
 			unsigned int _folio_nr_pages;
-			/* 4 byte gap here */
-	/* private: the union with struct page is transitional */
-			/* Fix THP_SWAP to not use tail->private */
-			unsigned long _private_1;
 #endif
+	/* private: the union with struct page is transitional */
 		};
 		struct page __page_1;
 	};
@@ -358,9 +355,6 @@ struct folio {
 	/* public: */
 			struct list_head _deferred_list;
 	/* private: the union with struct page is transitional */
-			unsigned long _avail_2a;
-			/* Fix THP_SWAP to not use tail->private */
-			unsigned long _private_2a;
 		};
 		struct page __page_2;
 	};
@@ -385,9 +379,6 @@ FOLIO_MATCH(memcg_data, memcg_data);
 			offsetof(struct page, pg) + sizeof(struct page))
 FOLIO_MATCH(flags, _flags_1);
 FOLIO_MATCH(compound_head, _head_1);
-#ifdef CONFIG_64BIT
-FOLIO_MATCH(private, _private_1);
-#endif
 #undef FOLIO_MATCH
 #define FOLIO_MATCH(pg, fl)						\
 	static_assert(offsetof(struct folio, fl) ==			\
@@ -396,7 +387,6 @@ FOLIO_MATCH(flags, _flags_2);
 FOLIO_MATCH(compound_head, _head_2);
 FOLIO_MATCH(flags, _flags_2a);
 FOLIO_MATCH(compound_head, _head_2a);
-FOLIO_MATCH(private, _private_2a);
 #undef FOLIO_MATCH
 
 /**
include/linux/swap.h +9 -0
index bb5adc604144..84fe0e94f5cd 100644
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)
 	return entry;
 }
 
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+	struct folio *folio = page_folio(page);
+	swp_entry_t entry = folio_swap_entry(folio);
+
+	entry.val += page - &folio->page;
+	return entry;
+}
+
 static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
 {
 	folio->private = (void *)entry.val;
mm/huge_memory.c +6 -9
index cc2f65f8cc62..c04702ae71d2 100644
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail,
 	page_tail->index = head->index + tail;
 
 	/*
-	 * page->private should not be set in tail pages with the exception
-	 * of swap cache pages that store the swp_entry_t in tail pages.
-	 * Fix up and warn once if private is unexpectedly set.
-	 *
-	 * What of 32-bit systems, on which folio->_pincount overlays
-	 * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and
-	 * pincount must be 0 for folio_ref_freeze() to have succeeded.
+	 * page->private should not be set in tail pages. Fix up and warn once
+	 * if private is unexpectedly set.
 	 */
-	if (!folio_test_swapcache(page_folio(head))) {
-		VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
+	if (unlikely(page_tail->private)) {
+		VM_WARN_ON_ONCE_PAGE(true, page_tail);
 		page_tail->private = 0;
 	}
+	if (PageSwapCache(head))
+		set_page_private(page_tail, (unsigned long)head->private + tail);
 
 	/* Page flags must be visible before we make the page non-compound. */
 	smp_wmb();
mm/memory.c +1 -1
index d003076b218d..ff13242c1589 100644
@@ -3882,7 +3882,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 		 * changed.
 		 */
 		if (unlikely(!folio_test_swapcache(folio) ||
-			     page_private(page) != entry.val))
+			     page_swap_entry(page).val != entry.val))
 			goto out_page;
 
 		/*
mm/rmap.c +1 -1
index 1f04debdc87a..ec7f8e6c9e48 100644
@@ -1647,7 +1647,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 			 */
 			dec_mm_counter(mm, mm_counter(&folio->page));
 		} else if (folio_test_anon(folio)) {
-			swp_entry_t entry = { .val = page_private(subpage) };
+			swp_entry_t entry = page_swap_entry(subpage);
 			pte_t swp_pte;
 			/*
 			 * Store the swap location in the pte.
mm/swap_state.c +3 -2
index 01f15139b7d9..2f2417810052 100644
@@ -100,6 +100,7 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
 
 	folio_ref_add(folio, nr);
 	folio_set_swapcache(folio);
+	folio_set_swap_entry(folio, entry);
 
 	do {
 		xas_lock_irq(&xas);
@@ -113,7 +114,6 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
 				if (shadowp)
 					*shadowp = old;
 			}
-			set_page_private(folio_page(folio, i), entry.val + i);
 			xas_store(&xas, folio);
 			xas_next(&xas);
 		}
@@ -154,9 +154,10 @@ void __delete_from_swap_cache(struct folio *folio,
 	for (i = 0; i < nr; i++) {
 		void *entry = xas_store(&xas, shadow);
 		VM_BUG_ON_PAGE(entry != folio, entry);
-		set_page_private(folio_page(folio, i), 0);
 		xas_next(&xas);
 	}
+	entry.val = 0;
+	folio_set_swap_entry(folio, entry);
 	folio_clear_swapcache(folio);
 	address_space->nrpages -= nr;
 	__node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
mm/swapfile.c +2 -2
index d46933adf789..bd9d904671b9 100644
@@ -3369,7 +3369,7 @@ struct swap_info_struct *swp_swap_info(swp_entry_t entry)
 
 struct swap_info_struct *page_swap_info(struct page *page)
 {
-	swp_entry_t entry = { .val = page_private(page) };
+	swp_entry_t entry = page_swap_entry(page);
 	return swp_swap_info(entry);
 }
 
@@ -3384,7 +3384,7 @@ EXPORT_SYMBOL_GPL(swapcache_mapping);
 
 pgoff_t __page_file_index(struct page *page)
 {
-	swp_entry_t swap = { .val = page_private(page) };
+	swp_entry_t swap = page_swap_entry(page);
 	return swp_offset(swap);
 }
 EXPORT_SYMBOL_GPL(__page_file_index);
Signature · 1 line
2.41.0
maintainer Acked-by on v1 1/4
#2 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

Quoted 6 lines · David Hildenbrand
arch/arm64/mm/mteswap.c +0 -0
index cd508ba80ab1..a31833e3ddc5 100644
@@ -33,8 +33,9 @@ int mte_save_tags(struct page *page)
 

arch/arm64/mm/mteswap.c int mte_save_tags(struct page *page) · line 41

Quoted 10 lines
arch/arm64/mm/mteswap.c +3 -2
 	mte_save_page_tags(page_address(page), tag_storage);
 
-	/* page_private contains the swap entry.val set in do_swap_page */
-	ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
+	/* lookup the swap entry.val from the page */
+	ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage,
+		       GFP_KERNEL);
 	if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
 		mte_free_tag_storage(tag_storage);
 		return xa_err(ret);

For arm64:

#3 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

In reply to David Hildenbrand
Quoted 88 lines · David Hildenbrand

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

---
 arch/arm64/mm/mteswap.c  |  5 +++--
 include/linux/mm_types.h | 12 +-----------
 include/linux/swap.h     |  9 +++++++++
 mm/huge_memory.c         | 15 ++++++---------
 mm/memory.c              |  2 +-
 mm/rmap.c                |  2 +-
 mm/swap_state.c          |  5 +++--
 mm/swapfile.c            |  4 ++--
 8 files changed, 26 insertions(+), 28 deletions(-)
arch/arm64/mm/mteswap.c +3 -2
index cd508ba80ab1..a31833e3ddc5 100644
@@ -33,8 +33,9 @@ int mte_save_tags(struct page *page)
 
        mte_save_page_tags(page_address(page), tag_storage);
 
-       /* page_private contains the swap entry.val set in do_swap_page */
-       ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
+       /* lookup the swap entry.val from the page */
+       ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage,
+                      GFP_KERNEL);
        if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
                mte_free_tag_storage(tag_storage);
                return xa_err(ret);
include/linux/mm_types.h +1 -11
index b9b6c88875b9..61361f1750c3 100644
@@ -333,11 +333,8 @@ struct folio {
                        atomic_t _pincount;
 #ifdef CONFIG_64BIT
                        unsigned int _folio_nr_pages;
-                       /* 4 byte gap here */
-       /* private: the union with struct page is transitional */
-                       /* Fix THP_SWAP to not use tail->private */
-                       unsigned long _private_1;
 #endif
+       /* private: the union with struct page is transitional */
                };
                struct page __page_1;
        };
@@ -358,9 +355,6 @@ struct folio {
        /* public: */
                        struct list_head _deferred_list;
        /* private: the union with struct page is transitional */
-                       unsigned long _avail_2a;
-                       /* Fix THP_SWAP to not use tail->private */
-                       unsigned long _private_2a;
                };
                struct page __page_2;
        };
@@ -385,9 +379,6 @@ FOLIO_MATCH(memcg_data, memcg_data);
                        offsetof(struct page, pg) + sizeof(struct page))
 FOLIO_MATCH(flags, _flags_1);
 FOLIO_MATCH(compound_head, _head_1);
-#ifdef CONFIG_64BIT
-FOLIO_MATCH(private, _private_1);
-#endif
 #undef FOLIO_MATCH
 #define FOLIO_MATCH(pg, fl)                                            \
        static_assert(offsetof(struct folio, fl) ==                     \
@@ -396,7 +387,6 @@ FOLIO_MATCH(flags, _flags_2);
 FOLIO_MATCH(compound_head, _head_2);
 FOLIO_MATCH(flags, _flags_2a);
 FOLIO_MATCH(compound_head, _head_2a);
-FOLIO_MATCH(private, _private_2a);
 #undef FOLIO_MATCH
 
 /**
include/linux/swap.h +0 -0
index bb5adc604144..84fe0e94f5cd 100644

include/linux/swap.h static inline swp_entry_t folio_swap_entry(struct folio *folio) · line 347

Quoted 9 lines
include/linux/swap.h +6 -0
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)
        return entry;
 }
 
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+       struct folio *folio = page_folio(page);
+       swp_entry_t entry = folio_swap_entry(folio);
+
+       entry.val += page - &folio->page;

Would it be better to use folio_page_idx() here?

Quoted 119 lines
include/linux/swap.h +3 -0
+       return entry;
+}
+
 static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
 {
        folio->private = (void *)entry.val;
mm/huge_memory.c +6 -9
index cc2f65f8cc62..c04702ae71d2 100644
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail,
        page_tail->index = head->index + tail;
 
        /*
-        * page->private should not be set in tail pages with the exception
-        * of swap cache pages that store the swp_entry_t in tail pages.
-        * Fix up and warn once if private is unexpectedly set.
-        *
-        * What of 32-bit systems, on which folio->_pincount overlays
-        * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and
-        * pincount must be 0 for folio_ref_freeze() to have succeeded.
+        * page->private should not be set in tail pages. Fix up and warn once
+        * if private is unexpectedly set.
         */
-       if (!folio_test_swapcache(page_folio(head))) {
-               VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
+       if (unlikely(page_tail->private)) {
+               VM_WARN_ON_ONCE_PAGE(true, page_tail);
                page_tail->private = 0;
        }
+       if (PageSwapCache(head))
+               set_page_private(page_tail, (unsigned long)head->private + tail);
 
        /* Page flags must be visible before we make the page non-compound. */
        smp_wmb();
mm/memory.c +1 -1
index d003076b218d..ff13242c1589 100644
@@ -3882,7 +3882,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
                 * changed.
                 */
                if (unlikely(!folio_test_swapcache(folio) ||
-                            page_private(page) != entry.val))
+                            page_swap_entry(page).val != entry.val))
                        goto out_page;
 
                /*
mm/rmap.c +1 -1
index 1f04debdc87a..ec7f8e6c9e48 100644
@@ -1647,7 +1647,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                         */
                        dec_mm_counter(mm, mm_counter(&folio->page));
                } else if (folio_test_anon(folio)) {
-                       swp_entry_t entry = { .val = page_private(subpage) };
+                       swp_entry_t entry = page_swap_entry(subpage);
                        pte_t swp_pte;
                        /*
                         * Store the swap location in the pte.
mm/swap_state.c +3 -2
index 01f15139b7d9..2f2417810052 100644
@@ -100,6 +100,7 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
 
        folio_ref_add(folio, nr);
        folio_set_swapcache(folio);
+       folio_set_swap_entry(folio, entry);
 
        do {
                xas_lock_irq(&xas);
@@ -113,7 +114,6 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
                                if (shadowp)
                                        *shadowp = old;
                        }
-                       set_page_private(folio_page(folio, i), entry.val + i);
                        xas_store(&xas, folio);
                        xas_next(&xas);
                }
@@ -154,9 +154,10 @@ void __delete_from_swap_cache(struct folio *folio,
        for (i = 0; i < nr; i++) {
                void *entry = xas_store(&xas, shadow);
                VM_BUG_ON_PAGE(entry != folio, entry);
-               set_page_private(folio_page(folio, i), 0);
                xas_next(&xas);
        }
+       entry.val = 0;
+       folio_set_swap_entry(folio, entry);
        folio_clear_swapcache(folio);
        address_space->nrpages -= nr;
        __node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
mm/swapfile.c +2 -3
index d46933adf789..bd9d904671b9 100644
@@ -3369,7 +3369,7 @@ struct swap_info_struct *swp_swap_info(swp_entry_t entry)
 
 struct swap_info_struct *page_swap_info(struct page *page)
 {
-       swp_entry_t entry = { .val = page_private(page) };
+       swp_entry_t entry = page_swap_entry(page);
        return swp_swap_info(entry);
 }
 
@@ -3384,7 +3384,7 @@ EXPORT_SYMBOL_GPL(swapcache_mapping);
 
 pgoff_t __page_file_index(struct page *page)
 {
-       swp_entry_t swap = { .val = page_private(page) };
+       swp_entry_t swap = page_swap_entry(page);
        return swp_offset(swap);
 }
 EXPORT_SYMBOL_GPL(__page_file_index);
--

2.41.0

#4 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

Quoted 9 lines

include/linux/swap.h static inline swp_entry_t folio_swap_entry(struct folio *folio) · line 347

Quoted 7 lines
include/linux/swap.h +6 -0
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+       struct folio *folio = page_folio(page);
+       swp_entry_t entry = folio_swap_entry(folio);
+
+       entry.val += page - &folio->page;

Would it be better to use folio_page_idx() here?

Sounds reasonable!

Signature · 3 lines
Cheers,

David / dhildenb
#5 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

In reply to David Hildenbrand
Quoted 11 lines · David Hildenbrand

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

---

@Andrew, the following change on top

From 2b1fd10ef07c6d47aa9cd0ce10445ab1e1b97361 Mon Sep 17 00:00:00 2001
From: David Hildenbrand <[email protected]>
Date: Tue, 22 Aug 2023 19:16:55 +0200
Subject: [PATCH] fixup: mm/swap: stop using page->private on tail pages for
THP_SWAP

Per Yoshry, use folio_page_idx(). It shouldn't make a difference for our (THP) use case, but it's certainly cleaner.

---
 include/linux/swap.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/swap.h +1 -1
index 84fe0e94f5cd..e5cf58a1cf9e 100644
@@ -344,7 +344,7 @@ static inline swp_entry_t page_swap_entry(struct page *page)
 	struct folio *folio = page_folio(page);
 	swp_entry_t entry = folio_swap_entry(folio);
 
-	entry.val += page - &folio->page;
+	entry.val += folio_page_idx(folio, page);
 	return entry;
 }
 
Signature · 7 lines
2.41.0


-- 
Cheers,

David / dhildenb
Reviewed-by on v1 1/4
#6 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

In reply to David Hildenbrand
Quoted 11 lines · David Hildenbrand

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

The mm part looks good to me (with the added fixup):

Minor nit below, not worth a respin, but perhaps if you respin anyway for something else.

Quoted 106 lines
---
 arch/arm64/mm/mteswap.c  |  5 +++--
 include/linux/mm_types.h | 12 +-----------
 include/linux/swap.h     |  9 +++++++++
 mm/huge_memory.c         | 15 ++++++---------
 mm/memory.c              |  2 +-
 mm/rmap.c                |  2 +-
 mm/swap_state.c          |  5 +++--
 mm/swapfile.c            |  4 ++--
 8 files changed, 26 insertions(+), 28 deletions(-)
arch/arm64/mm/mteswap.c +3 -2
index cd508ba80ab1..a31833e3ddc5 100644
@@ -33,8 +33,9 @@ int mte_save_tags(struct page *page)
 
        mte_save_page_tags(page_address(page), tag_storage);
 
-       /* page_private contains the swap entry.val set in do_swap_page */
-       ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
+       /* lookup the swap entry.val from the page */
+       ret = xa_store(&mte_pages, page_swap_entry(page).val, tag_storage,
+                      GFP_KERNEL);
        if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
                mte_free_tag_storage(tag_storage);
                return xa_err(ret);
include/linux/mm_types.h +1 -11
index b9b6c88875b9..61361f1750c3 100644
@@ -333,11 +333,8 @@ struct folio {
                        atomic_t _pincount;
 #ifdef CONFIG_64BIT
                        unsigned int _folio_nr_pages;
-                       /* 4 byte gap here */
-       /* private: the union with struct page is transitional */
-                       /* Fix THP_SWAP to not use tail->private */
-                       unsigned long _private_1;
 #endif
+       /* private: the union with struct page is transitional */
                };
                struct page __page_1;
        };
@@ -358,9 +355,6 @@ struct folio {
        /* public: */
                        struct list_head _deferred_list;
        /* private: the union with struct page is transitional */
-                       unsigned long _avail_2a;
-                       /* Fix THP_SWAP to not use tail->private */
-                       unsigned long _private_2a;
                };
                struct page __page_2;
        };
@@ -385,9 +379,6 @@ FOLIO_MATCH(memcg_data, memcg_data);
                        offsetof(struct page, pg) + sizeof(struct page))
 FOLIO_MATCH(flags, _flags_1);
 FOLIO_MATCH(compound_head, _head_1);
-#ifdef CONFIG_64BIT
-FOLIO_MATCH(private, _private_1);
-#endif
 #undef FOLIO_MATCH
 #define FOLIO_MATCH(pg, fl)                                            \
        static_assert(offsetof(struct folio, fl) ==                     \
@@ -396,7 +387,6 @@ FOLIO_MATCH(flags, _flags_2);
 FOLIO_MATCH(compound_head, _head_2);
 FOLIO_MATCH(flags, _flags_2a);
 FOLIO_MATCH(compound_head, _head_2a);
-FOLIO_MATCH(private, _private_2a);
 #undef FOLIO_MATCH
 
 /**
include/linux/swap.h +9 -0
index bb5adc604144..84fe0e94f5cd 100644
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)
        return entry;
 }
 
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+       struct folio *folio = page_folio(page);
+       swp_entry_t entry = folio_swap_entry(folio);
+
+       entry.val += page - &folio->page;
+       return entry;
+}
+
 static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
 {
        folio->private = (void *)entry.val;
mm/huge_memory.c +0 -6
index cc2f65f8cc62..c04702ae71d2 100644
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail,
        page_tail->index = head->index + tail;
 
        /*
-        * page->private should not be set in tail pages with the exception
-        * of swap cache pages that store the swp_entry_t in tail pages.
-        * Fix up and warn once if private is unexpectedly set.
-        *
-        * What of 32-bit systems, on which folio->_pincount overlays
-        * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and

mm/huge_memory.c static void __split_huge_page_tail(struct page *head, int tail, · line 2455

Quoted 10 lines
mm/huge_memory.c +4 -3
-        * pincount must be 0 for folio_ref_freeze() to have succeeded.
+        * page->private should not be set in tail pages. Fix up and warn once
+        * if private is unexpectedly set.
         */
-       if (!folio_test_swapcache(page_folio(head))) {
-               VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
+       if (unlikely(page_tail->private)) {
+               VM_WARN_ON_ONCE_PAGE(true, page_tail);
                page_tail->private = 0;
        }

Could probably save a couple of lines here:

if (VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail))

       page_tail->private = 0;
Quoted 89 lines
mm/huge_memory.c +2 -0
+       if (PageSwapCache(head))
+               set_page_private(page_tail, (unsigned long)head->private + tail);
 
        /* Page flags must be visible before we make the page non-compound. */
        smp_wmb();
mm/memory.c +1 -1
index d003076b218d..ff13242c1589 100644
@@ -3882,7 +3882,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
                 * changed.
                 */
                if (unlikely(!folio_test_swapcache(folio) ||
-                            page_private(page) != entry.val))
+                            page_swap_entry(page).val != entry.val))
                        goto out_page;
 
                /*
mm/rmap.c +1 -1
index 1f04debdc87a..ec7f8e6c9e48 100644
@@ -1647,7 +1647,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                         */
                        dec_mm_counter(mm, mm_counter(&folio->page));
                } else if (folio_test_anon(folio)) {
-                       swp_entry_t entry = { .val = page_private(subpage) };
+                       swp_entry_t entry = page_swap_entry(subpage);
                        pte_t swp_pte;
                        /*
                         * Store the swap location in the pte.
mm/swap_state.c +3 -2
index 01f15139b7d9..2f2417810052 100644
@@ -100,6 +100,7 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
 
        folio_ref_add(folio, nr);
        folio_set_swapcache(folio);
+       folio_set_swap_entry(folio, entry);
 
        do {
                xas_lock_irq(&xas);
@@ -113,7 +114,6 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
                                if (shadowp)
                                        *shadowp = old;
                        }
-                       set_page_private(folio_page(folio, i), entry.val + i);
                        xas_store(&xas, folio);
                        xas_next(&xas);
                }
@@ -154,9 +154,10 @@ void __delete_from_swap_cache(struct folio *folio,
        for (i = 0; i < nr; i++) {
                void *entry = xas_store(&xas, shadow);
                VM_BUG_ON_PAGE(entry != folio, entry);
-               set_page_private(folio_page(folio, i), 0);
                xas_next(&xas);
        }
+       entry.val = 0;
+       folio_set_swap_entry(folio, entry);
        folio_clear_swapcache(folio);
        address_space->nrpages -= nr;
        __node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
mm/swapfile.c +2 -3
index d46933adf789..bd9d904671b9 100644
@@ -3369,7 +3369,7 @@ struct swap_info_struct *swp_swap_info(swp_entry_t entry)
 
 struct swap_info_struct *page_swap_info(struct page *page)
 {
-       swp_entry_t entry = { .val = page_private(page) };
+       swp_entry_t entry = page_swap_entry(page);
        return swp_swap_info(entry);
 }
 
@@ -3384,7 +3384,7 @@ EXPORT_SYMBOL_GPL(swapcache_mapping);
 
 pgoff_t __page_file_index(struct page *page)
 {
-       swp_entry_t swap = { .val = page_private(page) };
+       swp_entry_t swap = page_swap_entry(page);
        return swp_offset(swap);
 }
 EXPORT_SYMBOL_GPL(__page_file_index);
--

2.41.0

#7 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

Quoted 8 lines · Yosry Ahmed
Quoted 7 lines · David Hildenbrand

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Quoted 8 lines
Quoted 4 lines

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

The mm part looks good to me (with the added fixup):

Thanks!

Quoted 48 lines
Quoted 35 lines

/**

include/linux/swap.h +9 -0
index bb5adc604144..84fe0e94f5cd 100644
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)
         return entry;
  }
 
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+       struct folio *folio = page_folio(page);
+       swp_entry_t entry = folio_swap_entry(folio);
+
+       entry.val += page - &folio->page;
+       return entry;
+}
+
  static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
  {
         folio->private = (void *)entry.val;
mm/huge_memory.c +0 -6
index cc2f65f8cc62..c04702ae71d2 100644
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail,
         page_tail->index = head->index + tail;
 
         /*
-        * page->private should not be set in tail pages with the exception
-        * of swap cache pages that store the swp_entry_t in tail pages.
-        * Fix up and warn once if private is unexpectedly set.
-        *
-        * What of 32-bit systems, on which folio->_pincount overlays
-        * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and

mm/huge_memory.c static void __split_huge_page_tail(struct page *head, int tail, · line 2455

Quoted 10 lines
mm/huge_memory.c +4 -3
-        * pincount must be 0 for folio_ref_freeze() to have succeeded.
+        * page->private should not be set in tail pages. Fix up and warn once
+        * if private is unexpectedly set.
          */
-       if (!folio_test_swapcache(page_folio(head))) {
-               VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
+       if (unlikely(page_tail->private)) {
+               VM_WARN_ON_ONCE_PAGE(true, page_tail);
                 page_tail->private = 0;
         }

Could probably save a couple of lines here:

Quoted 4 lines

if (VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail))

        page_tail->private = 0;

That would mean that we eventually compile out the runtime check

#define VM_WARN_ON_ONCE_PAGE(cond, page) BUILD_BUG_ON_INVALID(cond)

Signature · 3 lines
Cheers,

David / dhildenb
#8 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

Quoted 69 lines · David Hildenbrand
Quoted 8 lines · Yosry Ahmed
Quoted 7 lines · David Hildenbrand

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Quoted 8 lines
Quoted 4 lines

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

The mm part looks good to me (with the added fixup):

Thanks!

Quoted 48 lines
Quoted 35 lines

/**

include/linux/swap.h +9 -0
index bb5adc604144..84fe0e94f5cd 100644
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)
         return entry;
  }
 
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+       struct folio *folio = page_folio(page);
+       swp_entry_t entry = folio_swap_entry(folio);
+
+       entry.val += page - &folio->page;
+       return entry;
+}
+
  static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
  {
         folio->private = (void *)entry.val;
mm/huge_memory.c +0 -6
index cc2f65f8cc62..c04702ae71d2 100644
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail,
         page_tail->index = head->index + tail;
 
         /*
-        * page->private should not be set in tail pages with the exception
-        * of swap cache pages that store the swp_entry_t in tail pages.
-        * Fix up and warn once if private is unexpectedly set.
-        *
-        * What of 32-bit systems, on which folio->_pincount overlays
-        * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and

mm/huge_memory.c static void __split_huge_page_tail(struct page *head, int tail, · line 2455

Quoted 10 lines
mm/huge_memory.c +4 -3
-        * pincount must be 0 for folio_ref_freeze() to have succeeded.
+        * page->private should not be set in tail pages. Fix up and warn once
+        * if private is unexpectedly set.
          */
-       if (!folio_test_swapcache(page_folio(head))) {
-               VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
+       if (unlikely(page_tail->private)) {
+               VM_WARN_ON_ONCE_PAGE(true, page_tail);
                 page_tail->private = 0;
         }

Could probably save a couple of lines here:

Quoted 8 lines
Quoted 4 lines

if (VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail))

        page_tail->private = 0;

That would mean that we eventually compile out the runtime check

#define VM_WARN_ON_ONCE_PAGE(cond, page) BUILD_BUG_ON_INVALID(cond)

I thought the warning would be compiled out but not the check, my bad.

Quoted 6 lines

--
Cheers,

David / dhildenb

#9 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

Quoted 72 lines · Yosry Ahmed
Quoted 71 lines · David Hildenbrand
Quoted 8 lines · Yosry Ahmed
Quoted 7 lines · David Hildenbrand

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Quoted 8 lines
Quoted 4 lines

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

The mm part looks good to me (with the added fixup):

Thanks!

Quoted 50 lines
Quoted 35 lines
   /**
include/linux/swap.h +9 -0
index bb5adc604144..84fe0e94f5cd 100644
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)
          return entry;
   }
 
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+       struct folio *folio = page_folio(page);
+       swp_entry_t entry = folio_swap_entry(folio);
+
+       entry.val += page - &folio->page;
+       return entry;
+}
+
   static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
   {
          folio->private = (void *)entry.val;
mm/huge_memory.c +0 -6
index cc2f65f8cc62..c04702ae71d2 100644
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail,
          page_tail->index = head->index + tail;
 
          /*
-        * page->private should not be set in tail pages with the exception
-        * of swap cache pages that store the swp_entry_t in tail pages.
-        * Fix up and warn once if private is unexpectedly set.
-        *
-        * What of 32-bit systems, on which folio->_pincount overlays
-        * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and

mm/huge_memory.c static void __split_huge_page_tail(struct page *head, int tail, · line 2455

Quoted 10 lines
mm/huge_memory.c +4 -3
-        * pincount must be 0 for folio_ref_freeze() to have succeeded.
+        * page->private should not be set in tail pages. Fix up and warn once
+        * if private is unexpectedly set.
           */
-       if (!folio_test_swapcache(page_folio(head))) {
-               VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
+       if (unlikely(page_tail->private)) {
+               VM_WARN_ON_ONCE_PAGE(true, page_tail);
                  page_tail->private = 0;
          }

Could probably save a couple of lines here:

if (VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail))

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
         page_tail->private = 0;

That would mean that we eventually compile out the runtime check

#define VM_WARN_ON_ONCE_PAGE(cond, page) BUILD_BUG_ON_INVALID(cond)

I thought the warning would be compiled out but not the check, my bad.

I even remembered that VM_WARN_ON_ONCE and friends could/should not be used in conditionals.

But we do seem to have two users now:

 $ git grep "if (VM_WARN_ON"
mm/mmap.c:              if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
mm/mmap.c:              if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))

But they only do warning-related action, to dump the stack, the vma, ...

So if the warnings get compiled out, also all the other stuff gets compiled out as well, which makes sense here.

Signature · 3 lines
Cheers,

David / dhildenb
#10 lore raw

Re: [PATCH mm-unstable v1 1/4] mm/swap: stop using page->private on tail pages for THP_SWAP

Quoted 88 lines · David Hildenbrand
Quoted 72 lines · Yosry Ahmed
Quoted 71 lines · David Hildenbrand
Quoted 8 lines · Yosry Ahmed
Quoted 7 lines · David Hildenbrand

Let's stop using page->private on tail pages, making it possible to just unconditionally reuse that field in the tail pages of large folios.

The remaining usage of the private field for THP_SWAP is in the THP splitting code (mm/huge_memory.c), that we'll handle separately later.

Quoted 8 lines
Quoted 4 lines

Update the THP_SWAP documentation and sanity checks in mm_types.h and __split_huge_page_tail().

The mm part looks good to me (with the added fixup):

Thanks!

Quoted 50 lines
Quoted 35 lines
   /**
include/linux/swap.h +9 -0
index bb5adc604144..84fe0e94f5cd 100644
@@ -339,6 +339,15 @@ static inline swp_entry_t folio_swap_entry(struct folio *folio)
          return entry;
   }
 
+static inline swp_entry_t page_swap_entry(struct page *page)
+{
+       struct folio *folio = page_folio(page);
+       swp_entry_t entry = folio_swap_entry(folio);
+
+       entry.val += page - &folio->page;
+       return entry;
+}
+
   static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
   {
          folio->private = (void *)entry.val;
mm/huge_memory.c +0 -6
index cc2f65f8cc62..c04702ae71d2 100644
@@ -2446,18 +2446,15 @@ static void __split_huge_page_tail(struct page *head, int tail,
          page_tail->index = head->index + tail;
 
          /*
-        * page->private should not be set in tail pages with the exception
-        * of swap cache pages that store the swp_entry_t in tail pages.
-        * Fix up and warn once if private is unexpectedly set.
-        *
-        * What of 32-bit systems, on which folio->_pincount overlays
-        * head[1].private?  No problem: THP_SWAP is not enabled on 32-bit, and

mm/huge_memory.c static void __split_huge_page_tail(struct page *head, int tail, · line 2455

Quoted 10 lines
mm/huge_memory.c +4 -3
-        * pincount must be 0 for folio_ref_freeze() to have succeeded.
+        * page->private should not be set in tail pages. Fix up and warn once
+        * if private is unexpectedly set.
           */
-       if (!folio_test_swapcache(page_folio(head))) {
-               VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail);
+       if (unlikely(page_tail->private)) {
+               VM_WARN_ON_ONCE_PAGE(true, page_tail);
                  page_tail->private = 0;
          }

Could probably save a couple of lines here:

if (VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail))

Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
         page_tail->private = 0;

That would mean that we eventually compile out the runtime check

#define VM_WARN_ON_ONCE_PAGE(cond, page) BUILD_BUG_ON_INVALID(cond)

I thought the warning would be compiled out but not the check, my bad.

I even remembered that VM_WARN_ON_ONCE and friends could/should not be used in conditionals.

But we do seem to have two users now:

Quoted 8 lines
  $ git grep "if (VM_WARN_ON"
mm/mmap.c:              if (VM_WARN_ON_ONCE_MM(vma->vm_end != vmi_end, mm))
mm/mmap.c:              if (VM_WARN_ON_ONCE_MM(vma->vm_start != vmi_start, mm))

But they only do warning-related action, to dump the stack, the vma, ...

So if the warnings get compiled out, also all the other stuff gets compiled out as well, which makes sense here.

Funny enough, I did the same grep and immediately thought that since we have users of that, then it's okay (i.e the check wouldn't be compiled out). I wasn't thorough enough to actually check what they are doing :)

Quoted 6 lines

--
Cheers,

David / dhildenb

#11 lore raw

PATCH 2/4 mm/swap: use dedicated entry for swap in folio

In reply to David Hildenbrand

From: Matthew Wilcox <[email protected]>

Let's stop working on the private field and use an explicit swap field. We have to move the swp_entry_t typedef.

---
 include/linux/mm_types.h | 23 +++++++++++++----------
 include/linux/swap.h     |  5 ++---
 2 files changed, 15 insertions(+), 13 deletions(-)
include/linux/mm_types.h +13 -10
index 61361f1750c3..438a07854f8c 100644
@@ -259,6 +259,14 @@ static inline struct page *encoded_page_ptr(struct encoded_page *page)
  */
 #define  TAIL_MAPPING_REUSED_MAX  (2)
 
+/*
+ * A swap entry has to fit into a "unsigned long", as the entry is hidden
+ * in the "index" field of the swapper address space.
+ */
+typedef struct {
+	unsigned long val;
+} swp_entry_t;
+
 /**
  * struct folio - Represents a contiguous set of bytes.
  * @flags: Identical to the page flags.
@@ -269,7 +277,7 @@ static inline struct page *encoded_page_ptr(struct encoded_page *page)
  * @index: Offset within the file, in units of pages.  For anonymous memory,
  *    this is the index from the beginning of the mmap.
  * @private: Filesystem per-folio data (see folio_attach_private()).
- *    Used for swp_entry_t if folio_test_swapcache().
+ * @swap: Used for swp_entry_t if folio_test_swapcache().
  * @_mapcount: Do not access this member directly.  Use folio_mapcount() to
  *    find out how many times this folio is mapped by userspace.
  * @_refcount: Do not access this member directly.  Use folio_ref_count()
@@ -312,7 +320,10 @@ struct folio {
 			};
 			struct address_space *mapping;
 			pgoff_t index;
-			void *private;
+			union {
+				void *private;
+				swp_entry_t swap;
+			};
 			atomic_t _mapcount;
 			atomic_t _refcount;
 #ifdef CONFIG_MEMCG
@@ -1220,14 +1231,6 @@ enum tlb_flush_reason {
 	NR_TLB_FLUSH_REASONS,
 };
 
- /*
-  * A swap entry has to fit into a "unsigned long", as the entry is hidden
-  * in the "index" field of the swapper address space.
-  */
-typedef struct {
-	unsigned long val;
-} swp_entry_t;
-
 /**
  * enum fault_flag - Fault flag definitions.
  * @FAULT_FLAG_WRITE: Fault was a write fault.
include/linux/swap.h +2 -3
index 84fe0e94f5cd..82859a1944f5 100644
@@ -335,8 +335,7 @@ struct swap_info_struct {
 
 static inline swp_entry_t folio_swap_entry(struct folio *folio)
 {
-	swp_entry_t entry = { .val = page_private(&folio->page) };
-	return entry;
+	return folio->swap;
 }
 
 static inline swp_entry_t page_swap_entry(struct page *page)
@@ -350,7 +349,7 @@ static inline swp_entry_t page_swap_entry(struct page *page)
 
 static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
 {
-	folio->private = (void *)entry.val;
+	folio->swap = entry;
 }
 
 /* linux/mm/workingset.c */
Signature · 1 line
2.41.0
maintainer Reviewed-by on v1 2/4
#12 lore raw

Re: [PATCH mm-unstable v1 2/4] mm/swap: use dedicated entry for swap in folio

Chris

Quoted 95 lines · David Hildenbrand

From: Matthew Wilcox <[email protected]>

Let's stop working on the private field and use an explicit swap field. We have to move the swp_entry_t typedef.

---
 include/linux/mm_types.h | 23 +++++++++++++----------
 include/linux/swap.h     |  5 ++---
 2 files changed, 15 insertions(+), 13 deletions(-)
include/linux/mm_types.h +13 -10
index 61361f1750c3..438a07854f8c 100644
@@ -259,6 +259,14 @@ static inline struct page *encoded_page_ptr(struct encoded_page *page)
  */
 #define  TAIL_MAPPING_REUSED_MAX  (2)
 
+/*
+ * A swap entry has to fit into a "unsigned long", as the entry is hidden
+ * in the "index" field of the swapper address space.
+ */
+typedef struct {
+       unsigned long val;
+} swp_entry_t;
+
 /**
  * struct folio - Represents a contiguous set of bytes.
  * @flags: Identical to the page flags.
@@ -269,7 +277,7 @@ static inline struct page *encoded_page_ptr(struct encoded_page *page)
  * @index: Offset within the file, in units of pages.  For anonymous memory,
  *    this is the index from the beginning of the mmap.
  * @private: Filesystem per-folio data (see folio_attach_private()).
- *    Used for swp_entry_t if folio_test_swapcache().
+ * @swap: Used for swp_entry_t if folio_test_swapcache().
  * @_mapcount: Do not access this member directly.  Use folio_mapcount() to
  *    find out how many times this folio is mapped by userspace.
  * @_refcount: Do not access this member directly.  Use folio_ref_count()
@@ -312,7 +320,10 @@ struct folio {
                        };
                        struct address_space *mapping;
                        pgoff_t index;
-                       void *private;
+                       union {
+                               void *private;
+                               swp_entry_t swap;
+                       };
                        atomic_t _mapcount;
                        atomic_t _refcount;
 #ifdef CONFIG_MEMCG
@@ -1220,14 +1231,6 @@ enum tlb_flush_reason {
        NR_TLB_FLUSH_REASONS,
 };
 
- /*
-  * A swap entry has to fit into a "unsigned long", as the entry is hidden
-  * in the "index" field of the swapper address space.
-  */
-typedef struct {
-       unsigned long val;
-} swp_entry_t;
-
 /**
  * enum fault_flag - Fault flag definitions.
  * @FAULT_FLAG_WRITE: Fault was a write fault.
include/linux/swap.h +2 -4
index 84fe0e94f5cd..82859a1944f5 100644
@@ -335,8 +335,7 @@ struct swap_info_struct {
 
 static inline swp_entry_t folio_swap_entry(struct folio *folio)
 {
-       swp_entry_t entry = { .val = page_private(&folio->page) };
-       return entry;
+       return folio->swap;
 }
 
 static inline swp_entry_t page_swap_entry(struct page *page)
@@ -350,7 +349,7 @@ static inline swp_entry_t page_swap_entry(struct page *page)
 
 static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
 {
-       folio->private = (void *)entry.val;
+       folio->swap = entry;
 }
 
 /* linux/mm/workingset.c */
--

2.41.0

#13 lore raw

PATCH 3/4 mm/swap: inline folio_set_swap_entry() and folio_swap_entry()

In reply to David Hildenbrand

Let's simply work on the folio directly and remove the helpers.

---
 include/linux/swap.h | 12 +-----------
 mm/memory.c          |  2 +-
 mm/shmem.c           |  6 +++---
 mm/swap_state.c      |  7 +++----
 mm/swapfile.c        |  2 +-
 mm/util.c            |  2 +-
 mm/vmscan.c          |  2 +-
 mm/zswap.c           |  4 ++--
 8 files changed, 13 insertions(+), 24 deletions(-)
include/linux/swap.h +1 -11
index 82859a1944f5..603acf813873 100644
@@ -333,25 +333,15 @@ struct swap_info_struct {
 					   */
 };
 
-static inline swp_entry_t folio_swap_entry(struct folio *folio)
-{
-	return folio->swap;
-}
-
 static inline swp_entry_t page_swap_entry(struct page *page)
 {
 	struct folio *folio = page_folio(page);
-	swp_entry_t entry = folio_swap_entry(folio);
+	swp_entry_t entry = folio->swap;
 
 	entry.val += page - &folio->page;
 	return entry;
 }
 
-static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
-{
-	folio->swap = entry;
-}
-
 /* linux/mm/workingset.c */
 bool workingset_test_recent(void *shadow, bool file, bool *workingset);
 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
mm/memory.c +1 -1
index ff13242c1589..c51800dbfa9b 100644
@@ -3831,7 +3831,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 				folio_add_lru(folio);
 
 				/* To provide entry to swap_readpage() */
-				folio_set_swap_entry(folio, entry);
+				folio->swap = entry;
 				swap_readpage(page, true, NULL);
 				folio->private = NULL;
 			}
mm/shmem.c +3 -3
index 7a0c1e19d9f8..fc1afe9dfcfe 100644
@@ -1657,7 +1657,7 @@ static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
 	int error;
 
 	old = *foliop;
-	entry = folio_swap_entry(old);
+	entry = old->swap;
 	swap_index = swp_offset(entry);
 	swap_mapping = swap_address_space(entry);
 
@@ -1678,7 +1678,7 @@ static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
 	__folio_set_locked(new);
 	__folio_set_swapbacked(new);
 	folio_mark_uptodate(new);
-	folio_set_swap_entry(new, entry);
+	new->swap = entry;
 	folio_set_swapcache(new);
 
 	/*
@@ -1800,7 +1800,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
 	/* We have to do this with folio locked to prevent races */
 	folio_lock(folio);
 	if (!folio_test_swapcache(folio) ||
-	    folio_swap_entry(folio).val != swap.val ||
+	    folio->swap.val != swap.val ||
 	    !shmem_confirm_swap(mapping, index, swap)) {
 		error = -EEXIST;
 		goto unlock;
mm/swap_state.c +3 -4
index 2f2417810052..b3b14bd0dd64 100644
@@ -100,7 +100,7 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
 
 	folio_ref_add(folio, nr);
 	folio_set_swapcache(folio);
-	folio_set_swap_entry(folio, entry);
+	folio->swap = entry;
 
 	do {
 		xas_lock_irq(&xas);
@@ -156,8 +156,7 @@ void __delete_from_swap_cache(struct folio *folio,
 		VM_BUG_ON_PAGE(entry != folio, entry);
 		xas_next(&xas);
 	}
-	entry.val = 0;
-	folio_set_swap_entry(folio, entry);
+	folio->swap.val = 0;
 	folio_clear_swapcache(folio);
 	address_space->nrpages -= nr;
 	__node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
@@ -233,7 +232,7 @@ bool add_to_swap(struct folio *folio)
  */
 void delete_from_swap_cache(struct folio *folio)
 {
-	swp_entry_t entry = folio_swap_entry(folio);
+	swp_entry_t entry = folio->swap;
 	struct address_space *address_space = swap_address_space(entry);
 
 	xa_lock_irq(&address_space->i_pages);
mm/swapfile.c +1 -1
index bd9d904671b9..e52f486834eb 100644
@@ -1536,7 +1536,7 @@ static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
 
 static bool folio_swapped(struct folio *folio)
 {
-	swp_entry_t entry = folio_swap_entry(folio);
+	swp_entry_t entry = folio->swap;
 	struct swap_info_struct *si = _swap_info_get(entry);
 
 	if (!si)
mm/util.c +1 -1
index cde229b05eb3..f31e2ca62cfa 100644
@@ -764,7 +764,7 @@ struct address_space *folio_mapping(struct folio *folio)
 		return NULL;
 
 	if (unlikely(folio_test_swapcache(folio)))
-		return swap_address_space(folio_swap_entry(folio));
+		return swap_address_space(folio->swap);
 
 	mapping = folio->mapping;
 	if ((unsigned long)mapping & PAGE_MAPPING_FLAGS)
mm/vmscan.c +1 -1
index c7c149cb8d66..6f13394b112e 100644
@@ -1423,7 +1423,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
 	}
 
 	if (folio_test_swapcache(folio)) {
-		swp_entry_t swap = folio_swap_entry(folio);
+		swp_entry_t swap = folio->swap;
 
 		if (reclaimed && !mapping_exiting(mapping))
 			shadow = workingset_eviction(folio, target_memcg);
mm/zswap.c +2 -2
index 7300b98d4a03..412b1409a0d7 100644
@@ -1190,7 +1190,7 @@ static void zswap_fill_page(void *ptr, unsigned long value)
 
 bool zswap_store(struct folio *folio)
 {
-	swp_entry_t swp = folio_swap_entry(folio);
+	swp_entry_t swp = folio->swap;
 	int type = swp_type(swp);
 	pgoff_t offset = swp_offset(swp);
 	struct page *page = &folio->page;
@@ -1370,7 +1370,7 @@ bool zswap_store(struct folio *folio)
 
 bool zswap_load(struct folio *folio)
 {
-	swp_entry_t swp = folio_swap_entry(folio);
+	swp_entry_t swp = folio->swap;
 	int type = swp_type(swp);
 	pgoff_t offset = swp_offset(swp);
 	struct page *page = &folio->page;
Signature · 1 line
2.41.0
maintainer Reviewed-by on v1 3/4
#14 lore raw

Re: [PATCH mm-unstable v1 3/4] mm/swap: inline folio_set_swap_entry() and folio_swap_entry()

Chris

Quoted 188 lines · David Hildenbrand

Let's simply work on the folio directly and remove the helpers.

---
 include/linux/swap.h | 12 +-----------
 mm/memory.c          |  2 +-
 mm/shmem.c           |  6 +++---
 mm/swap_state.c      |  7 +++----
 mm/swapfile.c        |  2 +-
 mm/util.c            |  2 +-
 mm/vmscan.c          |  2 +-
 mm/zswap.c           |  4 ++--
 8 files changed, 13 insertions(+), 24 deletions(-)
include/linux/swap.h +1 -11
index 82859a1944f5..603acf813873 100644
@@ -333,25 +333,15 @@ struct swap_info_struct {
                                           */
 };
 
-static inline swp_entry_t folio_swap_entry(struct folio *folio)
-{
-       return folio->swap;
-}
-
 static inline swp_entry_t page_swap_entry(struct page *page)
 {
        struct folio *folio = page_folio(page);
-       swp_entry_t entry = folio_swap_entry(folio);
+       swp_entry_t entry = folio->swap;
 
        entry.val += page - &folio->page;
        return entry;
 }
 
-static inline void folio_set_swap_entry(struct folio *folio, swp_entry_t entry)
-{
-       folio->swap = entry;
-}
-
 /* linux/mm/workingset.c */
 bool workingset_test_recent(void *shadow, bool file, bool *workingset);
 void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
mm/memory.c +1 -1
index ff13242c1589..c51800dbfa9b 100644
@@ -3831,7 +3831,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
                                folio_add_lru(folio);
 
                                /* To provide entry to swap_readpage() */
-                               folio_set_swap_entry(folio, entry);
+                               folio->swap = entry;
                                swap_readpage(page, true, NULL);
                                folio->private = NULL;
                        }
mm/shmem.c +3 -3
index 7a0c1e19d9f8..fc1afe9dfcfe 100644
@@ -1657,7 +1657,7 @@ static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
        int error;
 
        old = *foliop;
-       entry = folio_swap_entry(old);
+       entry = old->swap;
        swap_index = swp_offset(entry);
        swap_mapping = swap_address_space(entry);
 
@@ -1678,7 +1678,7 @@ static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
        __folio_set_locked(new);
        __folio_set_swapbacked(new);
        folio_mark_uptodate(new);
-       folio_set_swap_entry(new, entry);
+       new->swap = entry;
        folio_set_swapcache(new);
 
        /*
@@ -1800,7 +1800,7 @@ static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
        /* We have to do this with folio locked to prevent races */
        folio_lock(folio);
        if (!folio_test_swapcache(folio) ||
-           folio_swap_entry(folio).val != swap.val ||
+           folio->swap.val != swap.val ||
            !shmem_confirm_swap(mapping, index, swap)) {
                error = -EEXIST;
                goto unlock;
mm/swap_state.c +3 -4
index 2f2417810052..b3b14bd0dd64 100644
@@ -100,7 +100,7 @@ int add_to_swap_cache(struct folio *folio, swp_entry_t entry,
 
        folio_ref_add(folio, nr);
        folio_set_swapcache(folio);
-       folio_set_swap_entry(folio, entry);
+       folio->swap = entry;
 
        do {
                xas_lock_irq(&xas);
@@ -156,8 +156,7 @@ void __delete_from_swap_cache(struct folio *folio,
                VM_BUG_ON_PAGE(entry != folio, entry);
                xas_next(&xas);
        }
-       entry.val = 0;
-       folio_set_swap_entry(folio, entry);
+       folio->swap.val = 0;
        folio_clear_swapcache(folio);
        address_space->nrpages -= nr;
        __node_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
@@ -233,7 +232,7 @@ bool add_to_swap(struct folio *folio)
  */
 void delete_from_swap_cache(struct folio *folio)
 {
-       swp_entry_t entry = folio_swap_entry(folio);
+       swp_entry_t entry = folio->swap;
        struct address_space *address_space = swap_address_space(entry);
 
        xa_lock_irq(&address_space->i_pages);
mm/swapfile.c +1 -1
index bd9d904671b9..e52f486834eb 100644
@@ -1536,7 +1536,7 @@ static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
 
 static bool folio_swapped(struct folio *folio)
 {
-       swp_entry_t entry = folio_swap_entry(folio);
+       swp_entry_t entry = folio->swap;
        struct swap_info_struct *si = _swap_info_get(entry);
 
        if (!si)
mm/util.c +1 -1
index cde229b05eb3..f31e2ca62cfa 100644
@@ -764,7 +764,7 @@ struct address_space *folio_mapping(struct folio *folio)
                return NULL;
 
        if (unlikely(folio_test_swapcache(folio)))
-               return swap_address_space(folio_swap_entry(folio));
+               return swap_address_space(folio->swap);
 
        mapping = folio->mapping;
        if ((unsigned long)mapping & PAGE_MAPPING_FLAGS)
mm/vmscan.c +1 -1
index c7c149cb8d66..6f13394b112e 100644
@@ -1423,7 +1423,7 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio,
        }
 
        if (folio_test_swapcache(folio)) {
-               swp_entry_t swap = folio_swap_entry(folio);
+               swp_entry_t swap = folio->swap;
 
                if (reclaimed && !mapping_exiting(mapping))
                        shadow = workingset_eviction(folio, target_memcg);
mm/zswap.c +2 -3
index 7300b98d4a03..412b1409a0d7 100644
@@ -1190,7 +1190,7 @@ static void zswap_fill_page(void *ptr, unsigned long value)
 
 bool zswap_store(struct folio *folio)
 {
-       swp_entry_t swp = folio_swap_entry(folio);
+       swp_entry_t swp = folio->swap;
        int type = swp_type(swp);
        pgoff_t offset = swp_offset(swp);
        struct page *page = &folio->page;
@@ -1370,7 +1370,7 @@ bool zswap_store(struct folio *folio)
 
 bool zswap_load(struct folio *folio)
 {
-       swp_entry_t swp = folio_swap_entry(folio);
+       swp_entry_t swp = folio->swap;
        int type = swp_type(swp);
        pgoff_t offset = swp_offset(swp);
        struct page *page = &folio->page;
--

2.41.0

#15 lore raw

PATCH 4/4 mm/huge_memory: work on folio->swap instead of page->private when splitting folio

In reply to David Hildenbrand

Let's work on folio->swap instead. While at it, use folio_test_anon() and folio_test_swapcache() -- the original folio remains valid even after splitting (but is then an order-0 folio).

We can probably convert a lot more to folios in that code, let's focus on folio->swap handling only for now.

---
 mm/huge_memory.c | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)
mm/huge_memory.c +15 -14
index c04702ae71d2..4465915711c3 100644
@@ -2401,10 +2401,16 @@ static void lru_add_page_tail(struct page *head, struct page *tail,
 	}
 }
 
-static void __split_huge_page_tail(struct page *head, int tail,
+static void __split_huge_page_tail(struct folio *folio, int tail,
 		struct lruvec *lruvec, struct list_head *list)
 {
+	struct page *head = &folio->page;
 	struct page *page_tail = head + tail;
+	/*
+	 * Careful: new_folio is not a "real" folio before we cleared PageTail.
+	 * Don't pass it around before clear_compound_head().
+	 */
+	struct folio *new_folio = (struct folio *)page_tail;
 
 	VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
 
@@ -2453,8 +2459,8 @@ static void __split_huge_page_tail(struct page *head, int tail,
 		VM_WARN_ON_ONCE_PAGE(true, page_tail);
 		page_tail->private = 0;
 	}
-	if (PageSwapCache(head))
-		set_page_private(page_tail, (unsigned long)head->private + tail);
+	if (folio_test_swapcache(folio))
+		new_folio->swap.val = folio->swap.val + tail;
 
 	/* Page flags must be visible before we make the page non-compound. */
 	smp_wmb();
@@ -2500,11 +2506,9 @@ static void __split_huge_page(struct page *page, struct list_head *list,
 	/* complete memcg works before add pages to LRU */
 	split_page_memcg(head, nr);
 
-	if (PageAnon(head) && PageSwapCache(head)) {
-		swp_entry_t entry = { .val = page_private(head) };
-
-		offset = swp_offset(entry);
-		swap_cache = swap_address_space(entry);
+	if (folio_test_anon(folio) && folio_test_swapcache(folio)) {
+		offset = swp_offset(folio->swap);
+		swap_cache = swap_address_space(folio->swap);
 		xa_lock(&swap_cache->i_pages);
 	}
 
@@ -2514,7 +2518,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
 	ClearPageHasHWPoisoned(head);
 
 	for (i = nr - 1; i >= 1; i--) {
-		__split_huge_page_tail(head, i, lruvec, list);
+		__split_huge_page_tail(folio, i, lruvec, list);
 		/* Some pages can be beyond EOF: drop them from page cache */
 		if (head[i].index >= end) {
 			struct folio *tail = page_folio(head + i);
@@ -2559,11 +2563,8 @@ static void __split_huge_page(struct page *page, struct list_head *list,
 
 	remap_page(folio, nr);
 
-	if (PageSwapCache(head)) {
-		swp_entry_t entry = { .val = page_private(head) };
-
-		split_swap_cluster(entry);
-	}
+	if (folio_test_swapcache(folio))
+		split_swap_cluster(folio->swap);
 
 	for (i = 0; i < nr; i++) {
 		struct page *subpage = head + i;
Signature · 1 line
2.41.0
maintainer Reviewed-by on v1 4/4
#16 lore raw

Re: [PATCH mm-unstable v1 4/4] mm/huge_memory: work on folio->swap instead of page->private when splitting folio

Hi David,

Chris

Chris

Quoted 88 lines · David Hildenbrand

Let's work on folio->swap instead. While at it, use folio_test_anon() and folio_test_swapcache() -- the original folio remains valid even after splitting (but is then an order-0 folio).

We can probably convert a lot more to folios in that code, let's focus on folio->swap handling only for now.

---
 mm/huge_memory.c | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)
mm/huge_memory.c +15 -15
index c04702ae71d2..4465915711c3 100644
@@ -2401,10 +2401,16 @@ static void lru_add_page_tail(struct page *head, struct page *tail,
        }
 }
 
-static void __split_huge_page_tail(struct page *head, int tail,
+static void __split_huge_page_tail(struct folio *folio, int tail,
                struct lruvec *lruvec, struct list_head *list)
 {
+       struct page *head = &folio->page;
        struct page *page_tail = head + tail;
+       /*
+        * Careful: new_folio is not a "real" folio before we cleared PageTail.
+        * Don't pass it around before clear_compound_head().
+        */
+       struct folio *new_folio = (struct folio *)page_tail;
 
        VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
 
@@ -2453,8 +2459,8 @@ static void __split_huge_page_tail(struct page *head, int tail,
                VM_WARN_ON_ONCE_PAGE(true, page_tail);
                page_tail->private = 0;
        }
-       if (PageSwapCache(head))
-               set_page_private(page_tail, (unsigned long)head->private + tail);
+       if (folio_test_swapcache(folio))
+               new_folio->swap.val = folio->swap.val + tail;
 
        /* Page flags must be visible before we make the page non-compound. */
        smp_wmb();
@@ -2500,11 +2506,9 @@ static void __split_huge_page(struct page *page, struct list_head *list,
        /* complete memcg works before add pages to LRU */
        split_page_memcg(head, nr);
 
-       if (PageAnon(head) && PageSwapCache(head)) {
-               swp_entry_t entry = { .val = page_private(head) };
-
-               offset = swp_offset(entry);
-               swap_cache = swap_address_space(entry);
+       if (folio_test_anon(folio) && folio_test_swapcache(folio)) {
+               offset = swp_offset(folio->swap);
+               swap_cache = swap_address_space(folio->swap);
                xa_lock(&swap_cache->i_pages);
        }
 
@@ -2514,7 +2518,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
        ClearPageHasHWPoisoned(head);
 
        for (i = nr - 1; i >= 1; i--) {
-               __split_huge_page_tail(head, i, lruvec, list);
+               __split_huge_page_tail(folio, i, lruvec, list);
                /* Some pages can be beyond EOF: drop them from page cache */
                if (head[i].index >= end) {
                        struct folio *tail = page_folio(head + i);
@@ -2559,11 +2563,8 @@ static void __split_huge_page(struct page *page, struct list_head *list,
 
        remap_page(folio, nr);
 
-       if (PageSwapCache(head)) {
-               swp_entry_t entry = { .val = page_private(head) };
-
-               split_swap_cluster(entry);
-       }
+       if (folio_test_swapcache(folio))
+               split_swap_cluster(folio->swap);
 
        for (i = 0; i < nr; i++) {
                struct page *subpage = head + i;
--

2.41.0