mm: Remember young bit for migration entries

We've got a bunch of special swap entries that stores PFN inside the swap offset fields. To fetch the PFN, normally the user just calls swp_offset() assuming that'll be the PFN.

Add a helper swp_offset_pfn() to fetch the PFN instead, fetching only the max possible length of a PFN on the host, meanwhile doing proper check with MAX_PHYSMEM_BITS to make sure the swap offsets can actually store the PFNs properly always using the BUILD_BUG_ON() in is_pfn_swap_entry().

One reason to do so is we never tried to sanitize whether swap offset can really fit for storing PFN. At the meantime, this patch also prepares us with the future possibility to store more information inside the swp offset field, so assuming "swp_offset(entry)" to be the PFN will not stand any more very soon.

Replace many of the swp_offset() callers to use swp_offset_pfn() where proper. Note that many of the existing users are not candidates for the replacement, e.g.:

(1) When the swap entry is not a pfn swap entry at all, or,
(2) when we wanna keep the whole swp_offset but only change the swp type.

For the latter, it can happen when fork() triggered on a write-migration swap entry pte, we may want to only change the migration type from write->read but keep the rest, so it's not "fetching PFN" but "changing swap type only". They're left aside so that when there're more information within the swp offset they'll be carried over naturally in those cases.

Since at it, dropping hwpoison_entry_to_pfn() because that's exactly what the new swp_offset_pfn() is about.

Signed-off-by Peter Xu <[email protected]>

---
 arch/arm64/mm/hugetlbpage.c |  2 +-
 include/linux/swapops.h     | 28 ++++++++++++++++++++++------
 mm/hmm.c                    |  2 +-
 mm/memory-failure.c         |  2 +-
 mm/page_vma_mapped.c        |  6 +++---
 5 files changed, 28 insertions(+), 12 deletions(-)

arch/arm64/mm/hugetlbpage.c +1 -1

index 7430060cb0d6..f897d40821dd 100644
@@ -242,7 +242,7 @@ static inline struct folio *hugetlb_swap_entry_to_folio(swp_entry_t entry)
 {
 	VM_BUG_ON(!is_migration_entry(entry) && !is_hwpoison_entry(entry));
 
-	return page_folio(pfn_to_page(swp_offset(entry)));
+	return page_folio(pfn_to_page(swp_offset_pfn(entry)));
 }
 
 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,

include/linux/swapops.h +8 -0

index a3d435bf9f97..5378f77860fb 100644
@@ -23,6 +23,14 @@
 #define SWP_TYPE_SHIFT	(BITS_PER_XA_VALUE - MAX_SWAPFILES_SHIFT)
 #define SWP_OFFSET_MASK	((1UL << SWP_TYPE_SHIFT) - 1)
 
+/*
+ * Definitions only for PFN swap entries (see is_pfn_swap_entry()).  To
+ * store PFN, we only need SWP_PFN_BITS bits.  Each of the pfn swap entries
+ * can use the extra bits to store other information besides PFN.
+ */
+#define SWP_PFN_BITS			(MAX_PHYSMEM_BITS - PAGE_SHIFT)
+#define SWP_PFN_MASK			((1UL << SWP_PFN_BITS) - 1)
+
 /* Clear all flags but only keep swp_entry_t related information */
 static inline pte_t pte_swp_clear_flags(pte_t pte)
 {

include/linux/swapops.h +4 -1

+	return swp_offset(entry) & SWP_PFN_MASK;
+}
+
 /* check whether a pte points to a swap entry */
 static inline int is_swap_pte(pte_t pte)
 {

@@ -369,7 +387,7 @@ static inline int pte_none_mostly(pte_t pte)
 
 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
 {
-	struct page *p = pfn_to_page(swp_offset(entry));
+	struct page *p = pfn_to_page(swp_offset_pfn(entry));
 
 	/*
 	 * Any use of migration entries may only occur while the

include/linux/swapops.h +1 -5

+
 	return is_migration_entry(entry) || is_device_private_entry(entry) ||
 	       is_device_exclusive_entry(entry);
 }

@@ -475,11 +496,6 @@ static inline int is_hwpoison_entry(swp_entry_t entry)
 	return swp_type(entry) == SWP_HWPOISON;
 }
 
-static inline unsigned long hwpoison_entry_to_pfn(swp_entry_t entry)
-{
-	return swp_offset(entry);
-}
-
 static inline void num_poisoned_pages_inc(void)
 {
 	atomic_long_inc(&num_poisoned_pages);

mm/hmm.c +1 -1

index f2aa63b94d9b..3850fb625dda 100644
@@ -253,7 +253,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
 			cpu_flags = HMM_PFN_VALID;
 			if (is_writable_device_private_entry(entry))
 				cpu_flags |= HMM_PFN_WRITE;
-			*hmm_pfn = swp_offset(entry) | cpu_flags;
+			*hmm_pfn = swp_offset_pfn(entry) | cpu_flags;
 			return 0;
 		}
 

mm/memory-failure.c +1 -1

index cc6fc9be8d22..e451219124dd 100644
@@ -632,7 +632,7 @@ static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift,
 		swp_entry_t swp = pte_to_swp_entry(pte);
 
 		if (is_hwpoison_entry(swp))
-			pfn = hwpoison_entry_to_pfn(swp);
+			pfn = swp_offset_pfn(swp);
 	}
 
 	if (!pfn || pfn != poisoned_pfn)

mm/page_vma_mapped.c +3 -3

index 8e9e574d535a..93e13fc17d3c 100644
@@ -86,7 +86,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw)
 		    !is_device_exclusive_entry(entry))
 			return false;
 
-		pfn = swp_offset(entry);
+		pfn = swp_offset_pfn(entry);
 	} else if (is_swap_pte(*pvmw->pte)) {
 		swp_entry_t entry;
 

@@ -96,7 +96,7 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw)
 		    !is_device_exclusive_entry(entry))
 			return false;
 
-		pfn = swp_offset(entry);
+		pfn = swp_offset_pfn(entry);
 	} else {
 		if (!pte_present(*pvmw->pte))
 			return false;

@@ -221,7 +221,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw)
 					return not_found(pvmw);
 				entry = pmd_to_swp_entry(pmde);
 				if (!is_migration_entry(entry) ||
-				    !check_pmd(swp_offset(entry), pvmw))
+				    !check_pmd(swp_offset_pfn(entry), pvmw))
 					return not_found(pvmw);
 				return true;
 			}

include/linux/swapops.h static inline pgoff_t swp_offset(swp_entry_t entry) · line 81

Quoted 7 lines

include/linux/swapops.h +7 -0

+/*
+ * This should only be called upon a pfn swap entry to get the PFN stored
+ * in the swap entry.  Please refers to is_pfn_swap_entry() for definition
+ * of pfn swap entry.
+ */
+static inline unsigned long swp_offset_pfn(swp_entry_t entry)
+{

Is it good to call is_pfn_swap_entry() here for debug that can be eliminated in the production kernel?

Quoted 14 lines

include/linux/swapops.h +4 -1

+	return swp_offset(entry) & SWP_PFN_MASK;
+}
+
 /* check whether a pte points to a swap entry */
 static inline int is_swap_pte(pte_t pte)
 {

@@ -369,7 +387,7 @@ static inline int pte_none_mostly(pte_t pte)
 
 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
 {
-	struct page *p = pfn_to_page(swp_offset(entry));
+	struct page *p = pfn_to_page(swp_offset_pfn(entry));
 

include/linux/swapops.h static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) · line 409

Quoted 8 lines

include/linux/swapops.h +2 -0

 	/*
 	 * Any use of migration entries may only occur while the

@@ -387,6 +405,9 @@ static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
  */
 static inline bool is_pfn_swap_entry(swp_entry_t entry)
 {
+	/* Make sure the swp offset can always store the needed fields */
+	BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);

	BUILD_BUG_ON(SWP_TYPE_SHIFT <= SWP_PFN_BITS);

Quoted 10 lines · Ying

include/linux/swapops.h static inline pgoff_t swp_offset(swp_entry_t entry) · line 81

Quoted 7 lines

include/linux/swapops.h +7 -0

+/*
+ * This should only be called upon a pfn swap entry to get the PFN stored
+ * in the swap entry.  Please refers to is_pfn_swap_entry() for definition
+ * of pfn swap entry.
+ */
+static inline unsigned long swp_offset_pfn(swp_entry_t entry)
+{

Is it good to call is_pfn_swap_entry() here for debug that can be eliminated in the production kernel?

Sure, I'll add a VM_BUG_ON() there in the next spin.

Quoted 14 lines

Quoted 14 lines

include/linux/swapops.h +4 -1

+	return swp_offset(entry) & SWP_PFN_MASK;
+}
+
 /* check whether a pte points to a swap entry */
 static inline int is_swap_pte(pte_t pte)
 {

@@ -369,7 +387,7 @@ static inline int pte_none_mostly(pte_t pte)
 
 static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
 {
-	struct page *p = pfn_to_page(swp_offset(entry));
+	struct page *p = pfn_to_page(swp_offset_pfn(entry));
 

Quoted 10 lines

include/linux/swapops.h static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry) · line 409

Quoted 8 lines

include/linux/swapops.h +2 -0

 	/*
 	 * Any use of migration entries may only occur while the

@@ -387,6 +405,9 @@ static inline struct page *pfn_swap_entry_to_page(swp_entry_t entry)
  */
 static inline bool is_pfn_swap_entry(swp_entry_t entry)
 {
+	/* Make sure the swp offset can always store the needed fields */
+	BUILD_BUG_ON(SWP_TYPE_SHIFT < SWP_PFN_BITS);

	BUILD_BUG_ON(SWP_TYPE_SHIFT <= SWP_PFN_BITS);

Logiclaly it's okay to have SWP_TYPE_SHIFT==SWP_PFN_BITS?

[Marking as RFC; only x86 is supported for now, plan to add a few more archs when there's a formal version]

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

Quoted 3 lines · Peter Xu

[Marking as RFC; only x86 is supported for now, plan to add a few more archs when there's a formal version]

Quoted 6 lines

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

Looks good to me.

I just wonder whether the order of the patches should be different. I always understood that separating the “enabling” patch from the others is not a good practice, since it complicates bisection. I guess it is more of a minor issue for such a small patch-set…

[Marking as RFC; only x86 is supported for now, plan to add a few more
archs when there's a formal version]

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

However that could lead to at least two problems:

(1) We lost the real hot/cold information while we could have persisted. That information shouldn't change even if the backing page is changed after the migration,

(2) There can be always extra overhead on the immediate next access to any migrated page, because hardware MMU needs cycles to set the young bit again (as long as the MMU supports).

Many of the recent upstream works showed that (2) is not something trivial and actually very measurable. In my test case, reading 1G chunk of memory
- jumping in page size intervals - could take 99ms just because of the extra setting on the young bit on a generic x86_64 system, comparing to 4ms if young set.

This issue is originally reported by Andrea Arcangeli.

Solution
========

To solve this problem, this patchset tries to remember the young bit in the migration entries and carry it over when recovering the ptes.

We have the chance to do so because in many systems the swap offset is not really fully used. Migration entries use swp offset to store PFN only, while the PFN is normally not as large as swp offset and normally smaller. It means we do have some free bits in swp offset that we can use to store things like young, and that's how this series tried to approach this problem.

One tricky thing here is even though we're embedding the information into swap entry which seems to be a very generic data structure, the number of bits that are free is still arch dependent. Not only because the size of swp_entry_t differs, but also due to the different layouts of swap ptes on different archs.

Here, this series requires specific arch to define an extra macro called __ARCH_SWP_OFFSET_BITS represents the size of swp offset. With this information, the swap logic can know whether there's extra bits to use, then it'll remember the young bits when possible. By default, it'll keep the old behavior of keeping all migrated pages cold.

Tests
=====

After the patchset applied, the immediate read access test [1] of above 1G chunk after migration can shrink from 99ms to 4ms. The test is done by moving 1G pages from node 0->1->0 then read it in page size jumps.

Currently __ARCH_SWP_OFFSET_BITS is only defined on x86 for this series and only tested on x86_64 with Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz.

Patch Layout
============

Patch 1: Add swp_offset_pfn() and apply to all pfn swap entries, we should
also stop treating swp_offset() as PFN anymore because it can contain more information starting from next patch.

Patch 2:  The core patch to remember young bit in swap offsets.
Patch 3:  A cleanup for x86 32 bits pgtable.h.
Patch 4:  Define __ARCH_SWP_OFFSET_BITS on x86, enable young bit for migration

Please review, thanks.

[1] https://github.com/xzpeter/clibs/blob/master/misc/swap-young.c

Peter Xu (4):
mm/swap: Add swp_offset_pfn() to fetch PFN from swap entry
mm: Remember young bit for page migrations mm/x86: Use SWP_TYPE_BITS in 3-level swap macros mm/x86: Define __ARCH_SWP_OFFSET_BITS

 arch/arm64/mm/hugetlbpage.c           |  2 +-
 arch/x86/include/asm/pgtable-2level.h |  6 ++
 arch/x86/include/asm/pgtable-3level.h | 15 +++--
 arch/x86/include/asm/pgtable_64.h     |  5 ++
 include/linux/swapops.h               | 85 +++++++++++++++++++++++++--
 mm/hmm.c                              |  2 +-
 mm/huge_memory.c                      | 10 +++-
 mm/memory-failure.c                   |  2 +-
 mm/migrate.c                          |  4 +-
 mm/migrate_device.c                   |  2 +
 mm/page_vma_mapped.c                  |  6 +-
 mm/rmap.c                             |  3 +-
 12 files changed, 122 insertions(+), 20 deletions(-)

[Marking as RFC; only x86 is supported for now, plan to add a few more
archs when there's a formal version]

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

However that could lead to at least two problems:

(1) We lost the real hot/cold information while we could have persisted. That information shouldn't change even if the backing page is changed after the migration,

(2) There can be always extra overhead on the immediate next access to any migrated page, because hardware MMU needs cycles to set the young bit again (as long as the MMU supports).

Many of the recent upstream works showed that (2) is not something trivial and actually very measurable. In my test case, reading 1G chunk of memory
- jumping in page size intervals - could take 99ms just because of the extra setting on the young bit on a generic x86_64 system, comparing to 4ms if young set.

This issue is originally reported by Andrea Arcangeli.

Solution
========

To solve this problem, this patchset tries to remember the young bit in the migration entries and carry it over when recovering the ptes.

We have the chance to do so because in many systems the swap offset is not really fully used. Migration entries use swp offset to store PFN only, while the PFN is normally not as large as swp offset and normally smaller. It means we do have some free bits in swp offset that we can use to store things like young, and that's how this series tried to approach this problem.

One tricky thing here is even though we're embedding the information into swap entry which seems to be a very generic data structure, the number of bits that are free is still arch dependent. Not only because the size of swp_entry_t differs, but also due to the different layouts of swap ptes on different archs.

Here, this series requires specific arch to define an extra macro called __ARCH_SWP_OFFSET_BITS represents the size of swp offset. With this information, the swap logic can know whether there's extra bits to use, then it'll remember the young bits when possible. By default, it'll keep the old behavior of keeping all migrated pages cold.

Tests
=====

After the patchset applied, the immediate read access test [1] of above 1G chunk after migration can shrink from 99ms to 4ms. The test is done by moving 1G pages from node 0->1->0 then read it in page size jumps.

Currently __ARCH_SWP_OFFSET_BITS is only defined on x86 for this series and only tested on x86_64 with Intel(R) Xeon(R) CPU E5-2630 v4 @ 2.20GHz.

Patch Layout
============

Patch 1: Add swp_offset_pfn() and apply to all pfn swap entries, we should
also stop treating swp_offset() as PFN anymore because it can contain more information starting from next patch.

Patch 2:  The core patch to remember young bit in swap offsets.
Patch 3:  A cleanup for x86 32 bits pgtable.h.
Patch 4:  Define __ARCH_SWP_OFFSET_BITS on x86, enable young bit for migration

Please review, thanks.

[1] https://github.com/xzpeter/clibs/blob/master/misc/swap-young.c

Peter Xu (4):
mm/swap: Add swp_offset_pfn() to fetch PFN from swap entry
mm: Remember young bit for page migrations mm/x86: Use SWP_TYPE_BITS in 3-level swap macros mm/x86: Define __ARCH_SWP_OFFSET_BITS

 arch/arm64/mm/hugetlbpage.c           |  2 +-
 arch/x86/include/asm/pgtable-2level.h |  6 ++
 arch/x86/include/asm/pgtable-3level.h | 15 +++--
 arch/x86/include/asm/pgtable_64.h     |  5 ++
 include/linux/swapops.h               | 85 +++++++++++++++++++++++++--
 mm/hmm.c                              |  2 +-
 mm/huge_memory.c                      | 10 +++-
 mm/memory-failure.c                   |  2 +-
 mm/migrate.c                          |  4 +-
 mm/migrate_device.c                   |  2 +
 mm/page_vma_mapped.c                  |  6 +-
 mm/rmap.c                             |  3 +-
 12 files changed, 122 insertions(+), 20 deletions(-)

If my understanding were correct, max_swapfile_size() provides a mechanism to identify the available bits with swp_entry_t and swap PTE considered. We may take advantage of that?

And according to commit 377eeaa8e11f ("x86/speculation/l1tf: Limit swap file size to MAX_PA/2"), the highest bit of swap offset needs to be 0 if L1TF mitigation is enabled.

Cced Andi for confirmation.

[Marking as RFC; only x86 is supported for now, plan to add a few more
archs when there's a formal version]

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

However that could lead to at least two problems:

(1) We lost the real hot/cold information while we could have persisted. That information shouldn't change even if the backing page is changed after the migration,

(2) There can be always extra overhead on the immediate next access to any migrated page, because hardware MMU needs cycles to set the young bit again (as long as the MMU supports).

Many of the recent upstream works showed that (2) is not something trivial and actually very measurable. In my test case, reading 1G chunk of memory
- jumping in page size intervals - could take 99ms just because of the extra setting on the young bit on a generic x86_64 system, comparing to 4ms if young set.

This issue is originally reported by Andrea Arcangeli.

Solution
========

To solve this problem, this patchset tries to remember the young bit in the migration entries and carry it over when recovering the ptes.

We have the chance to do so because in many systems the swap offset is not really fully used. Migration entries use swp offset to store PFN only, while the PFN is normally not as large as swp offset and normally smaller. It means we do have some free bits in swp offset that we can use to store things like young, and that's how this series tried to approach this problem.

One tricky thing here is even though we're embedding the information into swap entry which seems to be a very generic data structure, the number of bits that are free is still arch dependent. Not only because the size of swp_entry_t differs, but also due to the different layouts of swap ptes on different archs.

Here, this series requires specific arch to define an extra macro called __ARCH_SWP_OFFSET_BITS represents the size of swp offset. With this information, the swap logic can know whether there's extra bits to use, then it'll remember the young bits when possible. By default, it'll keep the old behavior of keeping all migrated pages cold.

Quoted 47 lines · Peter Xu

[Marking as RFC; only x86 is supported for now, plan to add a few more
archs when there's a formal version]

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

However that could lead to at least two problems:

(1) We lost the real hot/cold information while we could have persisted. That information shouldn't change even if the backing page is changed after the migration,

(2) There can be always extra overhead on the immediate next access to any migrated page, because hardware MMU needs cycles to set the young bit again (as long as the MMU supports).

Many of the recent upstream works showed that (2) is not something trivial and actually very measurable. In my test case, reading 1G chunk of memory
- jumping in page size intervals - could take 99ms just because of the extra setting on the young bit on a generic x86_64 system, comparing to 4ms if young set.

This issue is originally reported by Andrea Arcangeli.

Solution
========

To solve this problem, this patchset tries to remember the young bit in the migration entries and carry it over when recovering the ptes.

We have the chance to do so because in many systems the swap offset is not really fully used. Migration entries use swp offset to store PFN only, while the PFN is normally not as large as swp offset and normally smaller. It means we do have some free bits in swp offset that we can use to store things like young, and that's how this series tried to approach this problem.

One tricky thing here is even though we're embedding the information into swap entry which seems to be a very generic data structure, the number of bits that are free is still arch dependent. Not only because the size of swp_entry_t differs, but also due to the different layouts of swap ptes on different archs.

Quoted 6 lines

Here, this series requires specific arch to define an extra macro called __ARCH_SWP_OFFSET_BITS represents the size of swp offset. With this information, the swap logic can know whether there's extra bits to use, then it'll remember the young bits when possible. By default, it'll keep the old behavior of keeping all migrated pages cold.

I played with a similar idea when working on pte_swp_exclusive() but gave up, because it ended up looking too hacky. Looking at patch #2, I get the same feeling again. Kind of hacky.

If we mostly only care about x86_64, and it's a performance improvement after all, why not simply do it like pte_swp_mkexclusive/pte_swp_exclusive/ ... and reuse a spare PTE bit?

Quoted 56 lines · David Hildenbrand

Quoted 53 lines · Peter Xu

[Marking as RFC; only x86 is supported for now, plan to add a few more
archs when there's a formal version]

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

However that could lead to at least two problems:

(1) We lost the real hot/cold information while we could have persisted. That information shouldn't change even if the backing page is changed after the migration,

(2) There can be always extra overhead on the immediate next access to any migrated page, because hardware MMU needs cycles to set the young bit again (as long as the MMU supports).

Many of the recent upstream works showed that (2) is not something trivial and actually very measurable. In my test case, reading 1G chunk of memory
- jumping in page size intervals - could take 99ms just because of the extra setting on the young bit on a generic x86_64 system, comparing to 4ms if young set.

This issue is originally reported by Andrea Arcangeli.

Solution
========

To solve this problem, this patchset tries to remember the young bit in the migration entries and carry it over when recovering the ptes.

We have the chance to do so because in many systems the swap offset is not really fully used. Migration entries use swp offset to store PFN only, while the PFN is normally not as large as swp offset and normally smaller. It means we do have some free bits in swp offset that we can use to store things like young, and that's how this series tried to approach this problem.

One tricky thing here is even though we're embedding the information into swap entry which seems to be a very generic data structure, the number of bits that are free is still arch dependent. Not only because the size of swp_entry_t differs, but also due to the different layouts of swap ptes on different archs.

Here, this series requires specific arch to define an extra macro called __ARCH_SWP_OFFSET_BITS represents the size of swp offset. With this information, the swap logic can know whether there's extra bits to use, then it'll remember the young bits when possible. By default, it'll keep the old behavior of keeping all migrated pages cold.

Quoted 3 lines

I played with a similar idea when working on pte_swp_exclusive() but gave up, because it ended up looking too hacky. Looking at patch #2, I get the same feeling again. Kind of hacky.

Could you explain what's the "hacky" part you mentioned?

I used swap entry to avoid per-arch operations. I failed to figure out a common way to know swp offset length myself so unluckily in this RFC I still needed one macro per-arch. Ying's suggestion seems to be a good fit here to me to remove the last arch-specific dependency.

Quoted 6 lines

If we mostly only care about x86_64, and it's a performance improvement after all, why not simply do it like pte_swp_mkexclusive/pte_swp_exclusive/ ... and reuse a spare PTE bit?

Page migration works for most archs, I want to have it work for all archs that can easily benefit from it.

Besides I actually have a question on the anon exclusive bit in the swap
pte: since we have that anyway, why we need a specific migration type for anon exclusive pages? Can it be simply read migration entries with anon exclusive bit set?

Quoted 57 lines · Peter Xu

Quoted 56 lines · David Hildenbrand

Quoted 53 lines · Peter Xu

[Marking as RFC; only x86 is supported for now, plan to add a few more
archs when there's a formal version]

Problem
=======

When migrate a page, right now we always mark the migrated page as old. The reason could be that we don't really know whether the page is hot or cold, so we could have taken it a default negative assuming that's safer.

However that could lead to at least two problems:

(1) We lost the real hot/cold information while we could have persisted. That information shouldn't change even if the backing page is changed after the migration,

(2) There can be always extra overhead on the immediate next access to any migrated page, because hardware MMU needs cycles to set the young bit again (as long as the MMU supports).

Many of the recent upstream works showed that (2) is not something trivial and actually very measurable. In my test case, reading 1G chunk of memory
- jumping in page size intervals - could take 99ms just because of the extra setting on the young bit on a generic x86_64 system, comparing to 4ms if young set.

This issue is originally reported by Andrea Arcangeli.

Solution
========

To solve this problem, this patchset tries to remember the young bit in the migration entries and carry it over when recovering the ptes.

We have the chance to do so because in many systems the swap offset is not really fully used. Migration entries use swp offset to store PFN only, while the PFN is normally not as large as swp offset and normally smaller. It means we do have some free bits in swp offset that we can use to store things like young, and that's how this series tried to approach this problem.

One tricky thing here is even though we're embedding the information into swap entry which seems to be a very generic data structure, the number of bits that are free is still arch dependent. Not only because the size of swp_entry_t differs, but also due to the different layouts of swap ptes on different archs.

Here, this series requires specific arch to define an extra macro called __ARCH_SWP_OFFSET_BITS represents the size of swp offset. With this information, the swap logic can know whether there's extra bits to use, then it'll remember the young bits when possible. By default, it'll keep the old behavior of keeping all migrated pages cold.

Quoted 5 lines

Quoted 3 lines

I played with a similar idea when working on pte_swp_exclusive() but gave up, because it ended up looking too hacky. Looking at patch #2, I get the same feeling again. Kind of hacky.

Could you explain what's the "hacky" part you mentioned?

SWP_PFN_OFFSET_FREE_BITS :)

It's a PFN offset and we're mangling in random other bits. That's hacky IMHO.

I played with the idea of converting all code to store bits in addition to the type + offset. But that requires digging through a lot of arch code to teach that code about additional flags, so I discarded that idea when working on the COW fixes.

Quoted 6 lines

I used swap entry to avoid per-arch operations. I failed to figure out a common way to know swp offset length myself so unluckily in this RFC I still needed one macro per-arch. Ying's suggestion seems to be a good fit here to me to remove the last arch-specific dependency.

Instead of mangling this into the PFN offset and let the arch tell you which bits of the PFN offset are unused ... rather remove the bits from the offset and define them manually to have a certain meaning. That's exactly how pte_swp_mkexclusive/pte_swp_exclusive/ is supposed to be handled on architectures that want to support it.

I hope I could make it clearer what the hacky part is IMHO :)

Quoted 7 lines

Quoted 6 lines

If we mostly only care about x86_64, and it's a performance improvement after all, why not simply do it like pte_swp_mkexclusive/pte_swp_exclusive/ ... and reuse a spare PTE bit?

Quoted 2 lines

Page migration works for most archs, I want to have it work for all archs that can easily benefit from it.

Yet we only care about x86-64 IIUC regarding performance, just the way the dirty bit is handled?

Quoted 6 lines

Besides I actually have a question on the anon exclusive bit in the swap
pte: since we have that anyway, why we need a specific migration type for anon exclusive pages? Can it be simply read migration entries with anon exclusive bit set?

Not before all arch support pte_swp_mkexclusive/pte_swp_exclusive/.

As pte_swp_mkexclusive/pte_swp_exclusive/ only applies to actual swap PTEs, you could even reuse that bit for migration entries and get at alteast the most relevant 64bit architectures supported easily.

I don't think we only care about x86_64? Should other archs have the same issue as long as there's the hardware young bit?

Even without it, it'll affect page reclaim logic too, and that's also not x86 only.

Quoted 7 lines

Quoted 6 lines

Besides I actually have a question on the anon exclusive bit in the swap
pte: since we have that anyway, why we need a specific migration type for anon exclusive pages? Can it be simply read migration entries with anon exclusive bit set?

Quoted 5 lines

Not before all arch support pte_swp_mkexclusive/pte_swp_exclusive/.

As pte_swp_mkexclusive/pte_swp_exclusive/ only applies to actual swap PTEs, you could even reuse that bit for migration entries and get at alteast the most relevant 64bit architectures supported easily.

Yes, but I think having two mechanisms for the single problem can confuse people.

IIUC the swap bit is already defined in major archs anyway, and since anon exclusive bit is best-effort (or am I wrong?..), I won't worry too much on

archs outside x86/arm/ppc/s390 on having anon exclusive bit lost during migrations, because afaict the whole swap type of ANON_EXCLUSIVE_READ is only servicing that very minority.. which seems to be a pity to waste the

swp type on all archs even if the archs defined swp pte bits just for anon exclusive.

Quoted 5 lines

I don't think we only care about x86_64? Should other archs have the same issue as long as there's the hardware young bit?

Even without it, it'll affect page reclaim logic too, and that's also not x86 only.

Okay, reading the cover letter and looking at the code my understanding was that x86-64 is the real focus.

Quoted 7 lines

Quoted 7 lines

Quoted 6 lines

Besides I actually have a question on the anon exclusive bit in the swap
pte: since we have that anyway, why we need a specific migration type for anon exclusive pages? Can it be simply read migration entries with anon exclusive bit set?

Quoted 9 lines

Quoted 5 lines

Not before all arch support pte_swp_mkexclusive/pte_swp_exclusive/.

As pte_swp_mkexclusive/pte_swp_exclusive/ only applies to actual swap PTEs, you could even reuse that bit for migration entries and get at alteast the most relevant 64bit architectures supported easily.

Yes, but I think having two mechanisms for the single problem can confuse people.

It would be one bit with two different meanings depending on the swp type.

Quoted 2 lines

IIUC the swap bit is already defined in major archs anyway, and since anon exclusive bit is best-effort (or am I wrong?..), I won't worry too much on

It kind-of is best effort, but the goal is to have all archs support it.

... just like the young bit here?

Quoted 4 lines

archs outside x86/arm/ppc/s390 on having anon exclusive bit lost during migrations, because afaict the whole swap type of ANON_EXCLUSIVE_READ is only servicing that very minority.. which seems to be a pity to waste the

I have a big item on my todo list to support all, but I have different priorities right now.

If there is no free bit, simply steal one from the offset ... which is the same thing your approach would do, just in a different way, no?

Quoted 2 lines

swp type on all archs even if the archs defined swp pte bits just for anon exclusive.

Why do we care? We walk about one type not one bit.

Quoted 31 lines · David Hildenbrand

Quoted 5 lines

I don't think we only care about x86_64? Should other archs have the same issue as long as there's the hardware young bit?

Even without it, it'll affect page reclaim logic too, and that's also not x86 only.

Okay, reading the cover letter and looking at the code my understanding was that x86-64 is the real focus.

Quoted 7 lines

Quoted 7 lines

Quoted 6 lines

Besides I actually have a question on the anon exclusive bit in the swap
pte: since we have that anyway, why we need a specific migration type for anon exclusive pages? Can it be simply read migration entries with anon exclusive bit set?

Quoted 9 lines

Quoted 5 lines

Not before all arch support pte_swp_mkexclusive/pte_swp_exclusive/.

As pte_swp_mkexclusive/pte_swp_exclusive/ only applies to actual swap PTEs, you could even reuse that bit for migration entries and get at alteast the most relevant 64bit architectures supported easily.

Yes, but I think having two mechanisms for the single problem can confuse people.

It would be one bit with two different meanings depending on the swp type.

Quoted 2 lines

IIUC the swap bit is already defined in major archs anyway, and since anon exclusive bit is best-effort (or am I wrong?..), I won't worry too much on

Quoted 3 lines

It kind-of is best effort, but the goal is to have all archs support it.

... just like the young bit here?

Exactly, so I'm also wondering whether we can move the swp pte anon exclusive bit into swp entry. It just sounds weird to have them defined in two ways.

Quoted 12 lines

Quoted 5 lines

archs outside x86/arm/ppc/s390 on having anon exclusive bit lost during migrations, because afaict the whole swap type of ANON_EXCLUSIVE_READ is only servicing that very minority.. which seems to be a pity to waste the

I have a big item on my todo list to support all, but I have different priorities right now.

If there is no free bit, simply steal one from the offset ... which is the same thing your approach would do, just in a different way, no?

Quoted 4 lines

Quoted 2 lines

swp type on all archs even if the archs defined swp pte bits just for anon exclusive.

Why do we care? We walk about one type not one bit.

The swap type address space is still limited, I'd say we should save when possible. I believe people caring about swapping care about the limit of swap devices too. If the offset can keep it, I think it's better than the

swap type. De-dup either the type or the swap pte bit would be nicer, imho.

Quoted 32 lines · Peter Xu

Quoted 31 lines · David Hildenbrand

Quoted 5 lines

I don't think we only care about x86_64? Should other archs have the same issue as long as there's the hardware young bit?

Even without it, it'll affect page reclaim logic too, and that's also not x86 only.

Okay, reading the cover letter and looking at the code my understanding was that x86-64 is the real focus.

Quoted 7 lines

Quoted 7 lines

Quoted 6 lines

Besides I actually have a question on the anon exclusive bit in the swap
pte: since we have that anyway, why we need a specific migration type for anon exclusive pages? Can it be simply read migration entries with anon exclusive bit set?

Quoted 9 lines

Quoted 5 lines

Not before all arch support pte_swp_mkexclusive/pte_swp_exclusive/.

As pte_swp_mkexclusive/pte_swp_exclusive/ only applies to actual swap PTEs, you could even reuse that bit for migration entries and get at alteast the most relevant 64bit architectures supported easily.

Yes, but I think having two mechanisms for the single problem can confuse people.

It would be one bit with two different meanings depending on the swp type.

Quoted 2 lines

IIUC the swap bit is already defined in major archs anyway, and since anon exclusive bit is best-effort (or am I wrong?..), I won't worry too much on

Quoted 7 lines

Quoted 3 lines

It kind-of is best effort, but the goal is to have all archs support it.

... just like the young bit here?

Exactly, so I'm also wondering whether we can move the swp pte anon exclusive bit into swp entry. It just sounds weird to have them defined in two ways.

I'd argue it's just the swp vs. nonswp difference that are in fact two different concepts (device+offset vs. type+pte). And some dirty details how swp entries are actually used.

With swp entries you have to be very careful, for example, take a look at radix_to_swp_entry() and swp_to_radix_entry(). That made me refrain from touching anything inside actual swp entries and instead store it in the pte.

Quoted 13 lines

Quoted 13 lines

Quoted 6 lines

archs outside x86/arm/ppc/s390 on having anon exclusive bit lost during migrations, because afaict the whole swap type of ANON_EXCLUSIVE_READ is only servicing that very minority.. which seems to be a pity to waste the

I have a big item on my todo list to support all, but I have different priorities right now.

If there is no free bit, simply steal one from the offset ... which is the same thing your approach would do, just in a different way, no?

Quoted 8 lines

Quoted 4 lines

Quoted 2 lines

swp type on all archs even if the archs defined swp pte bits just for anon exclusive.

Why do we care? We walk about one type not one bit.

The swap type address space is still limited, I'd say we should save when possible. I believe people caring about swapping care about the limit of swap devices too. If the offset can keep it, I think it's better than the

Ehm, last time I did the math I came to the conclusion that nobody cares. Let me redo the math:

MAX_SWAPFILES = 1<<5 - 1 - 1 - 4 - 3 - 1 = 22

Which is the worst case right now with all kinds of oddity compiled in (sorry CONFIG_DEVICE_PRIVATE).

So far nobody complaint.

Quoted 3 lines

swap type. De-dup either the type or the swap pte bit would be nicer, imho.

If you manage bits in the pte manually, you might be able to get a better packing density, if bits are scattered around. Just take a look at the x86_64 location of _PAGE_SWP_EXCLUSIVE.

What I'm rooting for is something like

#define pte_nonswp_mkyoung pte_swp_mkexclusive

Eventually with some VM_BUG_ONs to make sure people call it on the right swp ptes.

If we ever want to get rid of SWP_MIGRATION_READ_EXCLUSIVE (so people can have 23 swap devices), and eventually have separate bits for both. For now it's not necessary.