support batch checking of references and unmapping for large folios

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Reviewed-by Harry Yoo <[email protected]>

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Reviewed-by Harry Yoo <[email protected]>

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
 include/linux/mmu_notifier.h |  9 +++++----
 include/linux/pgtable.h      | 35 +++++++++++++++++++++++++++++++++++
 mm/rmap.c                    | 28 +++++++++++++++++++++++++---
 3 files changed, 65 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +5 -4

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
        range->owner = owner;
 }
 
-#define ptep_clear_flush_young_notify(__vma, __address, __ptep)                \
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr)  \
 ({                                                                     \
        int __young;                                                    \
        struct vm_area_struct *___vma = __vma;                          \
        unsigned long ___address = __address;                           \
-       __young = ptep_clear_flush_young(___vma, ___address, __ptep);   \
+       unsigned int ___nr = __nr;                                      \
+       __young = clear_flush_young_ptes(___vma, ___address, __ptep, ___nr);    \
        __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,        \
                                                  ___address,           \
                                                  ___address +          \
-                                                       PAGE_SIZE);     \
+                                                 ___nr * PAGE_SIZE);   \
        __young;                                                        \
 })
 

@@ -650,7 +651,7 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
 
 #define mmu_notifier_range_update_to_read_only(r) false
 
-#define ptep_clear_flush_young_notify ptep_clear_flush_young
+#define clear_flush_young_ptes_notify clear_flush_young_ptes
 #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
 #define ptep_clear_young_notify ptep_test_and_clear_young
 #define pmdp_clear_young_notify pmdp_test_and_clear_young

include/linux/pgtable.h +24 -0

index 21b67d937555..a50df42a893f 100644
@@ -1068,6 +1068,41 @@ static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
 }
 #endif
 
+#ifndef clear_flush_young_ptes
+/**
+ * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same
+ *                         folio as old and flush the TLB.
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+               unsigned long addr, pte_t *ptep, unsigned int nr)
+{
+       int young = 0;
+
+       for (;;) {

Quoted 19 lines · Baolin Wang

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Quoted 3 lines

Reviewed-by Harry Yoo <[email protected]>

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

LGTM,

Reviewed-by Barry Song <[email protected]>

Quoted 79 lines

---
  include/linux/mmu_notifier.h |  9 +++++----
  include/linux/pgtable.h      | 35 +++++++++++++++++++++++++++++++++++
  mm/rmap.c                    | 28 +++++++++++++++++++++++++---
  3 files changed, 65 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +5 -4

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
         range->owner = owner;
  }
 
-#define ptep_clear_flush_young_notify(__vma, __address, __ptep)                \
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr)  \
  ({                                                                     \
         int __young;                                                    \
         struct vm_area_struct *___vma = __vma;                          \
         unsigned long ___address = __address;                           \
-       __young = ptep_clear_flush_young(___vma, ___address, __ptep);   \
+       unsigned int ___nr = __nr;                                      \
+       __young = clear_flush_young_ptes(___vma, ___address, __ptep, ___nr);    \
         __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,        \
                                                   ___address,           \
                                                   ___address +          \
-                                                       PAGE_SIZE);     \
+                                                 ___nr * PAGE_SIZE);   \
         __young;                                                        \
  })
 

@@ -650,7 +651,7 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
 
  #define mmu_notifier_range_update_to_read_only(r) false
 
-#define ptep_clear_flush_young_notify ptep_clear_flush_young
+#define clear_flush_young_ptes_notify clear_flush_young_ptes
  #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
  #define ptep_clear_young_notify ptep_test_and_clear_young
  #define pmdp_clear_young_notify pmdp_test_and_clear_young

include/linux/pgtable.h +31 -0

index 21b67d937555..a50df42a893f 100644
@@ -1068,6 +1068,41 @@ static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
  }
  #endif
 
+#ifndef clear_flush_young_ptes
+/**
+ * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same
+ *                         folio as old and flush the TLB.
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+               unsigned long addr, pte_t *ptep, unsigned int nr)
+{
+       int young = 0;
+
+       for (;;) {
+               young |= ptep_clear_flush_young(vma, addr, ptep);
+               if (--nr == 0)
+                       break;
+               ptep++;
+               addr += PAGE_SIZE;
+       }
+

include/linux/pgtable.h static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, · line 1104

Quoted 3 lines

include/linux/pgtable.h +3 -0

+       return young;
+}
+#endif

We might have an opportunity to batch the TLB synchronization, using flush_tlb_range() instead of calling flush_tlb_page() one by one. Not sure the benefit would be significant though, especially if only one entry among nr has the young bit set.

Quoted 21 lines · Barry Song

Quoted 19 lines · Baolin Wang

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Quoted 7 lines

Quoted 3 lines

Reviewed-by Harry Yoo <[email protected]>

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

LGTM,

Reviewed-by Barry Song <[email protected]>

Thanks.

Quoted 79 lines

Quoted 79 lines

---
  include/linux/mmu_notifier.h |  9 +++++----
  include/linux/pgtable.h      | 35 +++++++++++++++++++++++++++++++++++
  mm/rmap.c                    | 28 +++++++++++++++++++++++++---
  3 files changed, 65 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +5 -4

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
         range->owner = owner;
  }
 
-#define ptep_clear_flush_young_notify(__vma, __address, __ptep)                \
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr)  \
  ({                                                                     \
         int __young;                                                    \
         struct vm_area_struct *___vma = __vma;                          \
         unsigned long ___address = __address;                           \
-       __young = ptep_clear_flush_young(___vma, ___address, __ptep);   \
+       unsigned int ___nr = __nr;                                      \
+       __young = clear_flush_young_ptes(___vma, ___address, __ptep, ___nr);    \
         __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,        \
                                                   ___address,           \
                                                   ___address +          \
-                                                       PAGE_SIZE);     \
+                                                 ___nr * PAGE_SIZE);   \
         __young;                                                        \
  })
 

@@ -650,7 +651,7 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
 
  #define mmu_notifier_range_update_to_read_only(r) false
 
-#define ptep_clear_flush_young_notify ptep_clear_flush_young
+#define clear_flush_young_ptes_notify clear_flush_young_ptes
  #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
  #define ptep_clear_young_notify ptep_test_and_clear_young
  #define pmdp_clear_young_notify pmdp_test_and_clear_young

include/linux/pgtable.h +31 -0

index 21b67d937555..a50df42a893f 100644
@@ -1068,6 +1068,41 @@ static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
  }
  #endif
 
+#ifndef clear_flush_young_ptes
+/**
+ * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same
+ *                         folio as old and flush the TLB.
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+               unsigned long addr, pte_t *ptep, unsigned int nr)
+{
+       int young = 0;
+
+       for (;;) {
+               young |= ptep_clear_flush_young(vma, addr, ptep);
+               if (--nr == 0)
+                       break;
+               ptep++;
+               addr += PAGE_SIZE;
+       }
+

Quoted 8 lines

include/linux/pgtable.h static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, · line 1104

Quoted 3 lines

include/linux/pgtable.h +3 -0

+       return young;
+}
+#endif

We might have an opportunity to batch the TLB synchronization, using flush_tlb_range() instead of calling flush_tlb_page() one by one. Not sure the benefit would be significant though, especially if only one entry among nr has the young bit set.

Yes. In addition, this will involve many architectures’ implementations and their differing TLB flush mechanisms, so it’s difficult to make a reasonable per-architecture measurement. If any architecture has a more efficient flush method, I’d prefer to implement an architecture‑specific clear_flush_young_ptes().

Quoted 123 lines · Baolin Wang

Quoted 21 lines · Barry Song

Quoted 19 lines · Baolin Wang

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Quoted 7 lines

Quoted 3 lines

Reviewed-by Harry Yoo <[email protected]>

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

LGTM,

Reviewed-by Barry Song <[email protected]>

Thanks.

Quoted 79 lines

Quoted 79 lines

---
   include/linux/mmu_notifier.h |  9 +++++----
   include/linux/pgtable.h      | 35 +++++++++++++++++++++++++++++++++++
   mm/rmap.c                    | 28 +++++++++++++++++++++++++---
   3 files changed, 65 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +5 -4

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
          range->owner = owner;
   }
 
-#define ptep_clear_flush_young_notify(__vma, __address, __ptep)                \
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr)  \
   ({                                                                     \
          int __young;                                                    \
          struct vm_area_struct *___vma = __vma;                          \
          unsigned long ___address = __address;                           \
-       __young = ptep_clear_flush_young(___vma, ___address, __ptep);   \
+       unsigned int ___nr = __nr;                                      \
+       __young = clear_flush_young_ptes(___vma, ___address, __ptep, ___nr);    \
          __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,        \
                                                    ___address,           \
                                                    ___address +          \
-                                                       PAGE_SIZE);     \
+                                                 ___nr * PAGE_SIZE);   \
          __young;                                                        \
   })
 

@@ -650,7 +651,7 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
 
   #define mmu_notifier_range_update_to_read_only(r) false
 
-#define ptep_clear_flush_young_notify ptep_clear_flush_young
+#define clear_flush_young_ptes_notify clear_flush_young_ptes
   #define pmdp_clear_flush_young_notify pmdp_clear_flush_young
   #define ptep_clear_young_notify ptep_test_and_clear_young
   #define pmdp_clear_young_notify pmdp_test_and_clear_young

include/linux/pgtable.h +31 -0

index 21b67d937555..a50df42a893f 100644
@@ -1068,6 +1068,41 @@ static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
   }
   #endif
 
+#ifndef clear_flush_young_ptes
+/**
+ * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same
+ *                         folio as old and flush the TLB.
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+               unsigned long addr, pte_t *ptep, unsigned int nr)
+{
+       int young = 0;
+
+       for (;;) {
+               young |= ptep_clear_flush_young(vma, addr, ptep);
+               if (--nr == 0)
+                       break;
+               ptep++;
+               addr += PAGE_SIZE;
+       }
+

Quoted 8 lines

include/linux/pgtable.h static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, · line 1104

Quoted 3 lines

include/linux/pgtable.h +3 -0

+       return young;
+}
+#endif

We might have an opportunity to batch the TLB synchronization, using flush_tlb_range() instead of calling flush_tlb_page() one by one. Not sure the benefit would be significant though, especially if only one entry among nr has the young bit set.

Quoted 5 lines

Yes. In addition, this will involve many architectures’ implementations and their differing TLB flush mechanisms, so it’s difficult to make a reasonable per-architecture measurement. If any architecture has a more efficient flush method, I’d prefer to implement an architecture‑specific clear_flush_young_ptes().

Right! Since TLBI is usually quite expensive, I wonder if a generic implementation for architectures lacking clear_flush_young_ptes() might benefit from something like the below (just a very rough idea):

int clear_flush_young_ptes(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep, unsigned int nr)

{
         unsigned long curr_addr = addr;
         int young = 0;

         while (nr--) {
                 young |= ptep_test_and_clear_young(vma, curr_addr, ptep);
                 ptep++;
                 curr_addr += PAGE_SIZE;
         }

         if (young)
                 flush_tlb_range(vma, addr, curr_addr);
         return young;
}

Quoted 32 lines · Barry Song

Quoted 4 lines · Baolin Wang

Quoted 8 lines

Thanks.

Yes. In addition, this will involve many architectures’ implementations and their differing TLB flush mechanisms, so it’s difficult to make a reasonable per-architecture measurement. If any architecture has a more efficient flush method, I’d prefer to implement an architecture‑specific clear_flush_young_ptes().

Right! Since TLBI is usually quite expensive, I wonder if a generic implementation for architectures lacking clear_flush_young_ptes() might benefit from something like the below (just a very rough idea):

int clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) {
         unsigned long curr_addr = addr;
         int young = 0;

         while (nr--) {
                 young |= ptep_test_and_clear_young(vma, curr_addr, ptep);
                 ptep++;
                 curr_addr += PAGE_SIZE;
         }

Quoted 4 lines

         if (young)
                 flush_tlb_range(vma, addr, curr_addr); return young;
}

I understand your point. I’m concerned that I can’t test this patch on every architecture to validate the benefits. Anyway, let me try this on my X86 machine first.

Quoted 40 lines · Baolin Wang

Quoted 36 lines · Barry Song

Quoted 4 lines · Baolin Wang

Quoted 8 lines

Thanks.

Yes. In addition, this will involve many architectures’ implementations and their differing TLB flush mechanisms, so it’s difficult to make a reasonable per-architecture measurement. If any architecture has a more efficient flush method, I’d prefer to implement an architecture‑specific clear_flush_young_ptes().

Right! Since TLBI is usually quite expensive, I wonder if a generic implementation for architectures lacking clear_flush_young_ptes() might benefit from something like the below (just a very rough idea):

int clear_flush_young_ptes(struct vm_area_struct *vma,
                 unsigned long addr, pte_t *ptep, unsigned int nr)
{
         unsigned long curr_addr = addr;
         int young = 0;

         while (nr--) {
                 young |= ptep_test_and_clear_young(vma, curr_addr,
ptep);
                 ptep++;
                 curr_addr += PAGE_SIZE;
         }

         if (young)
                 flush_tlb_range(vma, addr, curr_addr); return young;
}

Quoted 3 lines

I understand your point. I’m concerned that I can’t test this patch on every architecture to validate the benefits. Anyway, let me try this on my X86 machine first.

In any case, please make that a follow-up patch :)

Quoted 9 lines · David Hildenbrand (Arm)

Quoted 6 lines · Baolin Wang

I understand your point. I’m concerned that I can’t test this patch on every architecture to validate the benefits. Anyway, let me try this on my X86 machine first.

In any case, please make that a follow-up patch :)

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when

     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep); }

Quoted 26 lines · Baolin Wang

Quoted 9 lines · David Hildenbrand (Arm)

Quoted 6 lines · Baolin Wang

I understand your point. I’m concerned that I can’t test this patch on every architecture to validate the benefits. Anyway, let me try this on my X86 machine first.

In any case, please make that a follow-up patch :)

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when

Quoted 8 lines

     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep); }

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Or some special flush_tlb_range() helper.

I agree that it requires more work.

--
Cheers,

David

Quoted 29 lines · David Hildenbrand (Arm)

Quoted 16 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when

Quoted 8 lines

     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep); }

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Quoted 3 lines

Or some special flush_tlb_range() helper.

I agree that it requires more work.

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Quoted 30 lines · Lorenzo Stoakes (Oracle)

Quoted 29 lines · David Hildenbrand (Arm)

Quoted 16 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when

Quoted 8 lines

     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep); }

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Quoted 6 lines

Quoted 3 lines

Or some special flush_tlb_range() helper.

I agree that it requires more work.

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Follow up!

--
Cheers,

David

Quoted 4 lines

Quoted 2 lines

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Follow up!

Ok good as in mm-stable now. Sadly means I don't get to review it but there we go.

Quoted 7 lines · On Wed, 25 Mar 2026 15:06:26 +0000 "Lorenzo Stoakes (Oracle)" <[email protected]>…

Quoted 4 lines

Quoted 2 lines

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Follow up!

Ok good as in mm-stable now. Sadly means I don't get to review it but there we go.

Well, this was sent 6 weeks ago, at v6!

But please go ahead. If review is very bad, there's git-rebase (happens sometimes) or some followup fix and a minor bisection hole (happens a bit more often).

And those followup fixes will trickle in all the way out to 7.1-rc7, via the hotfixes path. This happens a great deal - about half(?) of hotfixes pertain to material we added in the current -rc cycle.

Really I guess I'd like to know the timing of when mm-stable is taken so I can know the deadline for stuff like this.

Quoted 2 lines · On Wed, 25 Mar 2026 15:32:50 +0000 "Lorenzo Stoakes (Oracle)" <[email protected]>…

Really I guess I'd like to know the timing of when mm-stable is taken so I can know the deadline for stuff like this.

Yeah, as mentioned elsewhere I'll add some words into the series file to try to describe this.

But the start of the migration is quite variable (-rc4 through -rc6) so I'll aim to keep people updated in the weekly "mm.git review status" emails.

Quoted 8 lines · Andrew Morton

Quoted 2 lines · On Wed, 25 Mar 2026 15:32:50 +0000 "Lorenzo Stoakes (Oracle)" <[email protected]>…

Really I guess I'd like to know the timing of when mm-stable is taken so I can know the deadline for stuff like this.

Yeah, as mentioned elsewhere I'll add some words into the series file to try to describe this.

Quoted 4 lines

But the start of the migration is quite variable (-rc4 through -rc6) so I'll aim to keep people updated in the weekly "mm.git review status" emails.

Thanks. Wasn't aware that was a thing actually, but I see
https://lore.kernel.org/linux-mm/[email protected]/
now.

Quoted 9 lines · On Wed, 25 Mar 2026 16:28:18 +0000 "Lorenzo Stoakes (Oracle)" <[email protected]>…

Quoted 8 lines · Andrew Morton

Quoted 2 lines · On Wed, 25 Mar 2026 15:32:50 +0000 "Lorenzo Stoakes (Oracle)" <[email protected]>…

Really I guess I'd like to know the timing of when mm-stable is taken so I can know the deadline for stuff like this.

Yeah, as mentioned elsewhere I'll add some words into the series file to try to describe this.

Quoted 8 lines

Quoted 4 lines

But the start of the migration is quite variable (-rc4 through -rc6) so I'll aim to keep people updated in the weekly "mm.git review status" emails.

Thanks. Wasn't aware that was a thing actually, but I see
https://lore.kernel.org/linux-mm/[email protected]/
now.

Well now I'm all sad.

Yeah, I started doing this Feb 2, shall attempt to sustain this weekly after we've hit -rc4 ish. I expect I'll be adding more waffle to these as things occur to me. A general overview of where the tree is at.

I can Bcc: people if asked ;) lmk.

Quoted 28 lines · David Hildenbrand (Arm)

Quoted 27 lines · Lorenzo Stoakes (Oracle)

Quoted 26 lines · David Hildenbrand (Arm)

Quoted 24 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
               unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when
     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep);
}

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Or some special flush_tlb_range() helper.

I agree that it requires more work.

Quoted 4 lines

Quoted 2 lines

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Follow up!

Ok good as in mm-stable now. Sadly means I don't get to review it but there we go.

Quoted 29 lines · Lorenzo Stoakes (Oracle)

Quoted 28 lines · David Hildenbrand (Arm)

Quoted 27 lines · Lorenzo Stoakes (Oracle)

Quoted 26 lines · David Hildenbrand (Arm)

Quoted 24 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
/*
* On x86 CPUs, clearing the accessed bit without a TLB flush
* doesn't cause data corruption. [ It could cause incorrect
* page aging and the (mistaken) reclaim of hot pages, but the
* chance of that should be relatively low. ]
*
* So as a performance optimization don't flush the TLB when
* clearing the accessed bit, it will eventually be flushed by
* a context switch or a VM operation anyway. [ In the rare
* event of it not getting flushed for a long time the delay
* shouldn't really matter because there's no real memory
* pressure for swapout to react to. ]
*/
return ptep_test_and_clear_young(vma, address, ptep);
}

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Or some special flush_tlb_range() helper.

I agree that it requires more work.

(Sorry, David. I forgot to reply to your email because I've had a lot to sort out recently.)

Rather than adding more arch helpers (we already have plenty for the young flag check), I think we should try removing the TLB flush, as I mentioned to Barry[1]. MGLRU reclaim already skips the TLB flush, and it seems to work fine. What do you think?

Here are our previous attempts to remove the TLB flush:

My patch: https://lkml.org/lkml/2023/10/24/533
Barry's patch:
https://lore.kernel.org/lkml/[email protected]/

[1]
https://lore.kernel.org/all/[email protected]/

Quoted 29 lines · Lorenzo Stoakes (Oracle)

Quoted 28 lines · David Hildenbrand (Arm)

Quoted 27 lines · Lorenzo Stoakes (Oracle)

Quoted 26 lines · David Hildenbrand (Arm)

Quoted 24 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
               unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when
     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep);
}

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Or some special flush_tlb_range() helper.

I agree that it requires more work.

(Sorry, David. I forgot to reply to your email because I've had a lot to sort out recently.)

Rather than adding more arch helpers (we already have plenty for the young flag check), I think we should try removing the TLB flush, as I mentioned to Barry[1]. MGLRU reclaim already skips the TLB flush, and it seems to work fine. What do you think?

Here are our previous attempts to remove the TLB flush:

My patch: https://lkml.org/lkml/2023/10/24/533
Barry's patch:
https://lore.kernel.org/lkml/[email protected]/

[1] https://lore.kernel.org/all/[email protected]/

Quoted 3 lines

Quoted 3 lines

Quoted 2 lines

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Quoted 4 lines

Quoted 1 line

Follow up!

Ok good as in mm-stable now. Sadly means I don't get to review it but there we go.

Actually this patchset has already been merged upstream:)

Quoted 58 lines · Baolin Wang

Quoted 29 lines · Lorenzo Stoakes (Oracle)

Quoted 28 lines · David Hildenbrand (Arm)

Quoted 27 lines · Lorenzo Stoakes (Oracle)

Quoted 26 lines · David Hildenbrand (Arm)

Quoted 24 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
               unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when
     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep);
}

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Or some special flush_tlb_range() helper.

I agree that it requires more work.

(Sorry, David. I forgot to reply to your email because I've had a lot to sort out recently.)

Rather than adding more arch helpers (we already have plenty for the young flag check), I think we should try removing the TLB flush, as I mentioned to Barry[1]. MGLRU reclaim already skips the TLB flush, and it seems to work fine. What do you think?

Here are our previous attempts to remove the TLB flush:

My patch: https://lkml.org/lkml/2023/10/24/533
Barry's patch:
https://lore.kernel.org/lkml/[email protected]/

[1] https://lore.kernel.org/all/[email protected]/

Quoted 3 lines

Quoted 3 lines

Quoted 2 lines

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Quoted 6 lines

Quoted 4 lines

Quoted 1 line

Follow up!

Ok good as in mm-stable now. Sadly means I don't get to review it but there we go.

Actually this patchset has already been merged upstream:)

Err but this revision was sent _during_ the merge window...?

Was sent on 9th Feb on Monday in merge window week 1, with a functional change listed:

- Skip batched unmapping for uffd case, reported by Dev. Thanks.

And then sent in 2nd batch on 18th Feb (see [0]).

So we were ok with 1 week of 'testing' (does anybody actually test -next during the merge window? Was it even sent to -next?) for what appears to be a functional change?

And there was ongoing feedback on this and the v5 series (at [1])?

This doesn't really feel sane?

And now I'm confused as to whether mm-stable patches can collect tags, since presumably this was in mm-stable at the point this respin was done?

Maybe I'm missing something here but this doesn't feel like a sane process?

Quoted 59 lines · Lorenzo Stoakes (Oracle)

Quoted 58 lines · Baolin Wang

Quoted 29 lines · Lorenzo Stoakes (Oracle)

Quoted 28 lines · David Hildenbrand (Arm)

Quoted 27 lines · Lorenzo Stoakes (Oracle)

Quoted 26 lines · David Hildenbrand (Arm)

Quoted 24 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
               unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when
     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep);
}

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Or some special flush_tlb_range() helper.

I agree that it requires more work.

(Sorry, David. I forgot to reply to your email because I've had a lot to sort out recently.)

Rather than adding more arch helpers (we already have plenty for the young flag check), I think we should try removing the TLB flush, as I mentioned to Barry[1]. MGLRU reclaim already skips the TLB flush, and it seems to work fine. What do you think?

Here are our previous attempts to remove the TLB flush:

My patch: https://lkml.org/lkml/2023/10/24/533
Barry's patch:
https://lore.kernel.org/lkml/[email protected]/

[1] https://lore.kernel.org/all/[email protected]/

Quoted 3 lines

Quoted 3 lines

Quoted 2 lines

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Quoted 6 lines

Quoted 6 lines

Quoted 4 lines

Quoted 1 line

Follow up!

Ok good as in mm-stable now. Sadly means I don't get to review it but there we go.

Actually this patchset has already been merged upstream:)

Let me try to make things clear.

Quoted 5 lines

Err but this revision was sent _during_ the merge window...?

Was sent on 9th Feb on Monday in merge window week 1, with a functional change listed:

Quoted 7 lines

- Skip batched unmapping for uffd case, reported by Dev. Thanks.

And then sent in 2nd batch on 18th Feb (see [0]).

So we were ok with 1 week of 'testing' (does anybody actually test -next during the merge window? Was it even sent to -next?) for what appears to be a functional change?

I posted v5 on Dec 26th[0], and it collected quite a few Reviewed-by tags and sat in mm-unstable for testing.

Later, Dev reported a uffd-related issue (I hope you recall that discussion). I posted a fix[1] for it on Jan 16th, which Andrew accepted.

Since then, the v5 series (plus the fix) continued to be tested in mm-unstable. We kept it there mainly because David mentioned he wanted to review the series, so we were waiting for his time.

On Feb 9th, after returning from vacation, David reviewed the series (thanks, David!). I replied to and addressed all his comments, then posted v6 on the same day[2].

Additionally, v6 had no functional changes compared to v5 + the fix, and it mainly addressed some coding style issues pointed out by David. I also discussed this with David off-list, and since there were no functional changes, my expectation was that it could still make it into the merge window. That is why v6 was merged.

[0] https://lore.kernel.org/linux-mm/[email protected]/#t
[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/

Quoted 1 line

And there was ongoing feedback on this and the v5 series (at [1])?

Regarding the feedback on v5, I believe everything has been addressed.

Quoted 2 lines

This doesn't really feel sane?

Quoted 4 lines

And now I'm confused as to whether mm-stable patches can collect tags, since presumably this was in mm-stable at the point this respin was done?

Maybe I'm missing something here but this doesn't feel like a sane process?

Andrew, David, please correct me if I've missed anything. Also, please let me know if there's anything in the process that needs to be improved. Thanks.

Quoted 90 lines · Baolin Wang

Quoted 59 lines · Lorenzo Stoakes (Oracle)

Quoted 58 lines · Baolin Wang

Quoted 29 lines · Lorenzo Stoakes (Oracle)

Quoted 28 lines · David Hildenbrand (Arm)

Quoted 27 lines · Lorenzo Stoakes (Oracle)

Quoted 26 lines · David Hildenbrand (Arm)

Quoted 24 lines · Baolin Wang

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
               unsigned long address, pte_t *ptep)
{
    /*
     * On x86 CPUs, clearing the accessed bit without a TLB flush
     * doesn't cause data corruption. [ It could cause incorrect
     * page aging and the (mistaken) reclaim of hot pages, but the
     * chance of that should be relatively low. ]
     *
     * So as a performance optimization don't flush the TLB when
     * clearing the accessed bit, it will eventually be flushed by
     * a context switch or a VM operation anyway. [ In the rare
     * event of it not getting flushed for a long time the delay
     * shouldn't really matter because there's no real memory
     * pressure for swapout to react to. ]
     */
    return ptep_test_and_clear_young(vma, address, ptep);
}

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

You'd probably want an arch helper then, that tells you whether a flush_tlb_range() after ptep_test_and_clear_young() is required.

Or some special flush_tlb_range() helper.

I agree that it requires more work.

(Sorry, David. I forgot to reply to your email because I've had a lot to sort out recently.)

Rather than adding more arch helpers (we already have plenty for the young flag check), I think we should try removing the TLB flush, as I mentioned to Barry[1]. MGLRU reclaim already skips the TLB flush, and it seems to work fine. What do you think?

Here are our previous attempts to remove the TLB flush:

My patch: https://lkml.org/lkml/2023/10/24/533
Barry's patch:
https://lore.kernel.org/lkml/[email protected]/

[1] https://lore.kernel.org/all/[email protected]/

Quoted 3 lines

Quoted 3 lines

Quoted 2 lines

Sorry unclear here - does the series need more work or does a follow up patch need more work?

Quoted 6 lines

Quoted 6 lines

Quoted 4 lines

Quoted 1 line

Follow up!

Ok good as in mm-stable now. Sadly means I don't get to review it but there we go.

Actually this patchset has already been merged upstream:)

Let me try to make things clear.

Quoted 5 lines

Err but this revision was sent _during_ the merge window...?

Was sent on 9th Feb on Monday in merge window week 1, with a functional change listed:

Quoted 7 lines

- Skip batched unmapping for uffd case, reported by Dev. Thanks.

And then sent in 2nd batch on 18th Feb (see [0]).

So we were ok with 1 week of 'testing' (does anybody actually test -next during the merge window? Was it even sent to -next?) for what appears to be a functional change?

I posted v5 on Dec 26th[0], and it collected quite a few Reviewed-by tags and sat in mm-unstable for testing.

Later, Dev reported a uffd-related issue (I hope you recall that discussion). I posted a fix[1] for it on Jan 16th, which Andrew accepted.

Quoted 7 lines

Since then, the v5 series (plus the fix) continued to be tested in mm-unstable. We kept it there mainly because David mentioned he wanted to review the series, so we were waiting for his time.

On Feb 9th, after returning from vacation, David reviewed the series (thanks, David!). I replied to and addressed all his comments, then posted v6 on the same day[2].

OK thanks, I see that now.

I still don't think we should have made any changes _during_ the merge window, even if they were simple code quality things.

Changing patches then seems just crazy to me, as even code quality stuff can cause unexpected bugs, and now we're having upstream take it.

Also this speaks to -fix patches just being broken in general.

If you'd just respun with the fix as a v6, then we'd know 'v6 sent on 16th Jan addressed this' and there'd be no isssue.

Now v5 isn't v5, there's v5 and something-not-v5 and to have a sense of the testing you have to go read a bunch of email chains.

It also means change logs are now really inaccurate:

Changes from v5:
- Collect reviewed tags from Ryan, Harry and David. Thanks.
- Fix some coding style issues (per David).
- Skip batched unmapping for uffd case, reported by Dev. Thanks.

And that to me means 'v5 didn't have this, v6 does'.

And it's really hard to track timelines for testing.

Quoted 5 lines

Additionally, v6 had no functional changes compared to v5 + the fix, and it mainly addressed some coding style issues pointed out by David. I also discussed this with David off-list, and since there were no functional changes, my expectation was that it could still make it into the merge window. That is why v6 was merged.

Yeah, we still shouldn't have taken changes to a series DURING the merge window, it's just crazy.

Quoted 11 lines

[0] https://lore.kernel.org/linux-mm/[email protected]/#t
[1] https://lore.kernel.org/linux-mm/[email protected]/
[2] https://lore.kernel.org/all/[email protected]/

Quoted 1 line

And there was ongoing feedback on this and the v5 series (at [1])?

Regarding the feedback on v5, I believe everything has been addressed.

Quoted 2 lines

This doesn't really feel sane?

Quoted 8 lines

Quoted 4 lines

And now I'm confused as to whether mm-stable patches can collect tags, since presumably this was in mm-stable at the point this respin was done?

Maybe I'm missing something here but this doesn't feel like a sane process?

Andrew, David, please correct me if I've missed anything. Also, please let me know if there's anything in the process that needs to be improved. Thanks.

This isn't on you, it's about the process as a whole. We need clear rules about when changes will be accepted and when not.

And frankly I think we need to do away with fix patches as a whole based on this, or at least anything even vaguely non-trivial or that potentially impacts code.

Quoted 1 line

And there was ongoing feedback on this and the v5 series (at [1])?

Regarding the feedback on v5, I believe everything has been addressed.

Quoted 6 lines

This doesn't really feel sane?

And now I'm confused as to whether mm-stable patches can collect tags, since
presumably this was in mm-stable at the point this respin was done?

Quoted 2 lines

Maybe I'm missing something here but this doesn't feel like a sane process?

Andrew, David, please correct me if I've missed anything. Also, please let me know if there's anything in the process that needs to be improved. Thanks.

Quoted 41 lines · David Hildenbrand (Arm)

Quoted 40 lines · Baolin Wang

Quoted 36 lines · Barry Song

Quoted 4 lines · Baolin Wang

Quoted 8 lines

Thanks.

Yes. In addition, this will involve many architectures’ implementations and their differing TLB flush mechanisms, so it’s difficult to make a reasonable per-architecture measurement. If any architecture has a more efficient flush method, I’d prefer to implement an architecture‑specific clear_flush_young_ptes().

Right! Since TLBI is usually quite expensive, I wonder if a generic implementation for architectures lacking clear_flush_young_ptes() might benefit from something like the below (just a very rough idea):

int clear_flush_young_ptes(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep, unsigned int nr)

{
          unsigned long curr_addr = addr;
          int young = 0;

          while (nr--) {
                  young |= ptep_test_and_clear_young(vma, curr_addr,
ptep);
                  ptep++;
                  curr_addr += PAGE_SIZE;
          }

          if (young)
                  flush_tlb_range(vma, addr, curr_addr);
          return young;
}

Quoted 5 lines

Quoted 3 lines

I understand your point. I’m concerned that I can’t test this patch on every architecture to validate the benefits. Anyway, let me try this on my X86 machine first.

In any case, please make that a follow-up patch :)

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
/*
* On x86 CPUs, clearing the accessed bit without a TLB flush
* doesn't cause data corruption. [ It could cause incorrect
* page aging and the (mistaken) reclaim of hot pages, but the
* chance of that should be relatively low. ]
*
* So as a performance optimization don't flush the TLB when
* clearing the accessed bit, it will eventually be flushed by
* a context switch or a VM operation anyway. [ In the rare
* event of it not getting flushed for a long time the delay
* shouldn't really matter because there's no real memory
* pressure for swapout to react to. ]
*/
return ptep_test_and_clear_young(vma, address, ptep);
}

I don't have access to other architectures, so I think we can postpone this optimization unless someone is interested in optimizing the TLB flush.

Quoted 73 lines · Baolin Wang

Quoted 41 lines · David Hildenbrand (Arm)

Quoted 40 lines · Baolin Wang

Quoted 36 lines · Barry Song

Quoted 4 lines · Baolin Wang

Quoted 8 lines

Thanks.

Yes. In addition, this will involve many architectures’ implementations and their differing TLB flush mechanisms, so it’s difficult to make a reasonable per-architecture measurement. If any architecture has a more efficient flush method, I’d prefer to implement an architecture‑specific clear_flush_young_ptes().

Right! Since TLBI is usually quite expensive, I wonder if a generic implementation for architectures lacking clear_flush_young_ptes() might benefit from something like the below (just a very rough idea):

int clear_flush_young_ptes(struct vm_area_struct *vma,
unsigned long addr, pte_t *ptep, unsigned int nr)

{
           unsigned long curr_addr = addr;
           int young = 0;

           while (nr--) {
                   young |= ptep_test_and_clear_young(vma, curr_addr,
ptep);
                   ptep++;
                   curr_addr += PAGE_SIZE;
           }

           if (young)
                   flush_tlb_range(vma, addr, curr_addr);
           return young;
}

Quoted 5 lines

Quoted 3 lines

I understand your point. I’m concerned that I can’t test this patch on every architecture to validate the benefits. Anyway, let me try this on my X86 machine first.

In any case, please make that a follow-up patch :)

Sure. However, after investigating RISC‑V and x86, I found that ptep_clear_flush_young() does not flush the TLB on these architectures:

int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
/*
* On x86 CPUs, clearing the accessed bit without a TLB flush
* doesn't cause data corruption. [ It could cause incorrect
* page aging and the (mistaken) reclaim of hot pages, but the
* chance of that should be relatively low. ]
*
* So as a performance optimization don't flush the TLB when
* clearing the accessed bit, it will eventually be flushed by
* a context switch or a VM operation anyway. [ In the rare
* event of it not getting flushed for a long time the delay
* shouldn't really matter because there's no real memory
* pressure for swapout to react to. ]
*/
return ptep_test_and_clear_young(vma, address, ptep);
}

Quoted 2 lines

I don't have access to other architectures, so I think we can postpone this optimization unless someone is interested in optimizing the TLB flush.

The comment is interesting. I think it likely applies to most architectures, including ARM64. The main reason ARM64 doesn’t use this approach is probably that it can issue tlbi_nosync and then rely on a final dsb to ensure all invalidations are completed— and tlbi_nosync itself is relatively cheap.

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

W/o patchset:
real 0m1.518s
user 0m0.000s
sys 0m1.518s

W/ patchset:
real 0m1.018s
user 0m0.000s
sys 0m1.018s

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  arch/arm64/include/asm/pgtable.h | 11 +++++++++++
  1 file changed, 11 insertions(+)

arch/arm64/include/asm/pgtable.h +0 -0

index 3dabf5ea17fa..a17eb8a76788 100644

Quoted 32 lines · Baolin Wang

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

W/o patchset:
real    0m1.518s
user    0m0.000s
sys    0m1.518s

W/ patchset:
real    0m1.018s
user    0m0.000s
sys    0m1.018s

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  arch/arm64/include/asm/pgtable.h | 11 +++++++++++

  1 file changed, 11 insertions(+)

arch/arm64/include/asm/pgtable.h +0 -0

index 3dabf5ea17fa..a17eb8a76788 100644

arch/arm64/include/asm/pgtable.h static inline int ptep_clear_flush_young(struct · line 1846

Quoted 9 lines

arch/arm64/include/asm/pgtable.h +6 -0

@@ -1838,6 +1838,17 @@ static inline int ptep_clear_flush_young(struct 
 vm_area_struct *vma,
       return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
   }
+#define clear_flush_young_ptes clear_flush_young_ptes
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+                     unsigned long addr, pte_t *ptep,
+                     unsigned int nr)
+{
+    if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))

I guess similar cases where we should never end up with non-present ptes should be updated accordingly.

ptep_test_and_clear_young(), for example, should never be called on non- present ptes.

Reviewed-by David Hildenbrand (Arm) <[email protected]>

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

W/o patchset:

real    0m1.518s
user    0m0.000s
sys     0m1.518s

W/ patchset:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
 arch/arm64/include/asm/pgtable.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

arch/arm64/include/asm/pgtable.h +0 -0

index 3dabf5ea17fa..a17eb8a76788 100644
@@ -1838,6 +1838,17 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
        return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
 }
 

arch/arm64/include/asm/pgtable.h +1 -1

+
 #define wrprotect_ptes wrprotect_ptes
 static __always_inline void wrprotect_ptes(struct mm_struct *mm,
                                unsigned long addr, pte_t *ptep, unsigned int nr)
--

2.47.3

Quoted 22 lines · Baolin Wang

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

W/o patchset:

real    0m1.518s
user    0m0.000s
sys     0m1.518s

W/ patchset:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

Quoted 2 lines

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

Reviewed-by Barry Song <[email protected]>

Quoted 13 lines

---
  arch/arm64/include/asm/pgtable.h | 11 +++++++++++
  1 file changed, 11 insertions(+)

arch/arm64/include/asm/pgtable.h +0 -0

index 3dabf5ea17fa..a17eb8a76788 100644
@@ -1838,6 +1838,17 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
         return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
  }
 

arch/arm64/include/asm/pgtable.h static inline int ptep_clear_flush_young(struct vm_area_struct *vma, · line 1850

Quoted 10 lines

arch/arm64/include/asm/pgtable.h +10 -0

+#define clear_flush_young_ptes clear_flush_young_ptes
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+                                        unsigned long addr, pte_t *ptep,
+                                        unsigned int nr)
+{
+       if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
+               return __ptep_clear_flush_young(vma, addr, ptep);
+
+       return contpte_clear_flush_young_ptes(vma, addr, ptep, nr);
+}

A similar question arises here:

If nr = 4 for 16KB large folios and one of those entries is young, we end up flushing the TLB for all 4 PTEs.

If all four entries are young, we win; if only one is young, it seems we flush 3 redundant pages. but arm64 has TLB coalescing, so maybe they are just one TLB?

Quoted 24 lines · Barry Song

Quoted 22 lines · Baolin Wang

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

W/o patchset:

real    0m1.518s
user    0m0.000s
sys     0m1.518s

W/ patchset:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

Quoted 4 lines

Quoted 2 lines

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

Reviewed-by Barry Song <[email protected]>

Thanks Barry. But this series has been upstreamed, I can not add your reviewed tag.

Quoted 26 lines

Quoted 13 lines

---
  arch/arm64/include/asm/pgtable.h | 11 +++++++++++
  1 file changed, 11 insertions(+)

arch/arm64/include/asm/pgtable.h +0 -0

index 3dabf5ea17fa..a17eb8a76788 100644
@@ -1838,6 +1838,17 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
         return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
  }
 

arch/arm64/include/asm/pgtable.h static inline int ptep_clear_flush_young(struct vm_area_struct *vma, · line 1850

Quoted 10 lines

arch/arm64/include/asm/pgtable.h +10 -0

+#define clear_flush_young_ptes clear_flush_young_ptes
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+                                        unsigned long addr, pte_t *ptep,
+                                        unsigned int nr)
+{
+       if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
+               return __ptep_clear_flush_young(vma, addr, ptep);
+
+       return contpte_clear_flush_young_ptes(vma, addr, ptep, nr);
+}

A similar question arises here:

Quoted 6 lines

If nr = 4 for 16KB large folios and one of those entries is young, we end up flushing the TLB for all 4 PTEs.

If all four entries are young, we win; if only one is young, it seems we flush 3 redundant pages. but arm64 has TLB coalescing, so maybe they are just one TLB?

We discussed a similar issue in the previous thread [1], and I quote some comments from Ryan:

"
My concern was the opportunity cost of evicting the entries for all the non-accessed parts of the folio from the TLB. But of course, I'm talking nonsense because the architecture does not allow caching non-accessed entries in the TLB.
"

[1]
https://lore.kernel.org/all/[email protected]/

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Note that it's not ready to support batched unmapping for uffd case, so let's still fallback to per-page unmapping for the uffd case.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:
real 0m0.249s
user 0m0.000s
sys 0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Reviewed-by Harry Yoo <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Similar to folio_referenced_one(), we can also apply batched unmapping for large file folios to optimize the performance of file folio reclamation. By supporting batched checking of the young flags, flushing TLB entries, and unmapping, I can observed a significant performance improvements in my performance tests for file folios reclamation. Please check the performance data in the commit message of each patch.

Quoted 10 lines · On Mon, 9 Feb 2026 22:07:23 +0800 Baolin Wang <[email protected]>…

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Quoted 7 lines

Similar to folio_referenced_one(), we can also apply batched unmapping for large file folios to optimize the performance of file folio reclamation. By supporting batched checking of the young flags, flushing TLB entries, and unmapping, I can observed a significant performance improvements in my performance tests for file folios reclamation. Please check the performance data in the commit message of each patch.

Thanks, I updated mm.git to this version. Below is how v6 altered mm.git.

I notice that this fix:

https://lore.kernel.org/all/[email protected]/T/#u

was not carried forward. Was this deliberate?

Also, regarding the 80-column tricks in folio_referenced_one(): we're allowed to do this ;)

				unsigned long end_addr;
				unsigned int max_nr;

				end_addr = pmd_addr_end(address, vma->vm_end);
				max_nr = (end_addr - address) >> PAGE_SHIFT;

  arch/arm64/include/asm/pgtable.h |    2 +-
  include/linux/pgtable.h          |   16 ++++++++++------
  mm/rmap.c                        |    9 +++------
  3 files changed, 14 insertions(+), 13 deletions(-)

arch/arm64/include/asm/pgtable.h renamed from arch/arm64/include/asm/pgtable.h~b +1 -1

@@ -1843,7 +1843,7 @@ static inline int clear_flush_young_ptes
  					 unsigned long addr, pte_t *ptep,
  					 unsigned int nr)
  {
-	if (likely(nr == 1 && !pte_valid_cont(__ptep_get(ptep))))
+	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
  		return __ptep_clear_flush_young(vma, addr, ptep);
 
  	return contpte_clear_flush_young_ptes(vma, addr, ptep, nr);

include/linux/pgtable.h renamed from include/linux/pgtable.h~b +10 -6

@@ -1070,8 +1070,8 @@ static inline void wrprotect_ptes(struct
 
  #ifndef clear_flush_young_ptes
  /**
- * clear_flush_young_ptes - Clear the access bit and perform a TLB flush for PTEs
- *			    that map consecutive pages of the same folio.
+ * clear_flush_young_ptes - Mark PTEs that map consecutive pages of the same
+ *			    folio as old and flush the TLB.
   * @vma: The virtual memory area the pages are mapped into.
   * @addr: Address the first page is mapped at.
   * @ptep: Page table pointer for the first entry.

@@ -1087,13 +1087,17 @@ static inline void wrprotect_ptes(struct
   * pages that belong to the same folio.  The PTEs are all in the same PMD.
   */
  static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
-					 unsigned long addr, pte_t *ptep,
-					 unsigned int nr)
+		unsigned long addr, pte_t *ptep, unsigned int nr)
  {
-	int i, young = 0;
+	int young = 0;
 
-	for (i = 0; i < nr; ++i, ++ptep, addr += PAGE_SIZE)
+	for (;;) {
  		young |= ptep_clear_flush_young(vma, addr, ptep);
+		if (--nr == 0)
+			break;
+		ptep++;
+		addr += PAGE_SIZE;
+	}
 
  	return young;
  }

mm/rmap.c renamed from mm/rmap.c~b +3 -6

@@ -963,10 +963,8 @@ static bool folio_referenced_one(struct
  				referenced++;
  		} else if (pvmw.pte) {
  			if (folio_test_large(folio)) {
-				unsigned long end_addr =
-					pmd_addr_end(address, vma->vm_end);
-				unsigned int max_nr =
-					(end_addr - address) >> PAGE_SHIFT;
+				unsigned long end_addr = pmd_addr_end(address, vma->vm_end);
+				unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT;
  				pte_t pteval = ptep_get(pvmw.pte);
 
  				nr = folio_pte_batch(folio, pvmw.pte,

@@ -974,8 +972,7 @@ static bool folio_referenced_one(struct
  			}
 
  			ptes += nr;
-			if (clear_flush_young_ptes_notify(vma, address,
-						pvmw.pte, nr))
+			if (clear_flush_young_ptes_notify(vma, address, pvmw.pte, nr))
  				referenced++;
  			/* Skip the batched PTEs */
  			pvmw.pte += nr - 1;

_