support batch checking of references and unmapping for large folios

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  include/linux/mmu_notifier.h |  9 +++++----
  include/linux/pgtable.h      | 31 +++++++++++++++++++++++++++++++
  mm/rmap.c                    | 31 ++++++++++++++++++++++++++++---
  3 files changed, 64 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +1 -2

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
  	range->owner = owner;
  }
 
-#define ptep_clear_flush_young_notify(__vma, __address, __ptep)		\
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr)	\
  ({									\
  	int __young;							\
  	struct vm_area_struct *___vma = __vma;				\
  	unsigned long ___address = __address;				\
-	__young = ptep_clear_flush_young(___vma, ___address, __ptep);	\

+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *
+ * Context: The caller holds the page table lock. The PTEs map consecutive
+ * pages that belong to the same folio. The PTEs are all in the same PMD.
+ */

include/linux/mmu_notifier.h +4 -0

+	return young;
+}
+#endif
+
  /*
   * On some architectures hardware does not set page access bit when accessing
   * memory page, it is responsibility of software setting this bit. It brings

mm/rmap.c +2 -0

index e805ddc5a27b..985ab0b085ba 100644
@@ -828,9 +828,11 @@ static bool folio_referenced_one(struct folio *folio,
  	struct folio_referenced_arg *pra = arg;
  	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
  	int ptes = 0, referenced = 0;
+	unsigned int nr;
 
  	while (page_vma_mapped_walk(&pvmw)) {
  		address = pvmw.address;
+		nr = 1;
 
  		if (vma->vm_flags & VM_LOCKED) {
  			ptes++;

mm/rmap.c +3 -0

  				referenced++;
+			/* Skip the batched PTEs */
+			pvmw.pte += nr - 1;
+			pvmw.address += (nr - 1) * PAGE_SIZE;
  		} else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
  			if (pmdp_clear_flush_young_notify(vma, address,
  						pvmw.pmd))

@@ -887,7 +904,15 @@ static bool folio_referenced_one(struct folio *folio,
  			WARN_ON_ONCE(1);
  		}
 

Quoted 45 lines · Baolin Wang

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  include/linux/mmu_notifier.h |  9 +++++---- include/linux/pgtable.h      | 31 +++++++++++++++++++++++++++++++ mm/rmap.c                    | 31 ++++++++++++++++++++++++++++---

  3 files changed, 64 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +3 -2

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
       range->owner = owner;
   }
-#define ptep_clear_flush_young_notify(__vma, __address, __ptep)        \
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, 
 __nr)    \
   ({                                    \
       int __young;                            \
       struct vm_area_struct *___vma = __vma;                \
       unsigned long ___address = __address;                \
-    __young = ptep_clear_flush_young(___vma, ___address, __ptep);    \
+    unsigned int ___nr = __nr;                    \
+    __young = clear_flush_young_ptes(___vma, ___address, __ptep, 
 ___nr);    \
       __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,    \

include/linux/mmu_notifier.h static inline void mmu_notifier_range_init_owner( · line 530

Quoted 6 lines

include/linux/mmu_notifier.h +1 -1

                             ___address,        \
                             ___address +        \
-                            PAGE_SIZE);    \
+                          ___nr * PAGE_SIZE);    \
       __young;                            \
   })

Man that's ugly, Not your fault, but can this possibly be turned into an inline function in a follow-up patch.

Quoted 5 lines

include/linux/mmu_notifier.h +3 -0

+#ifndef clear_flush_young_ptes
+/**
+ * clear_flush_young_ptes - Clear the access bit and perform a TLB 

flush for PTEs
+ *                that map consecutive pages of the same folio.

With clear_young_dirty_ptes() description in mind, this should probably be "Mark PTEs that map consecutive pages of the same folio as clean and flush the TLB" ?

Quoted 13 lines

+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *

Quoted 8 lines

+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma, +                     unsigned long addr, pte_t *ptep,
+                     unsigned int nr)

Two-tab alignment on second+ line like all similar functions here.

include/linux/mmu_notifier.h

Quoted 6 lines

include/linux/mmu_notifier.h +6 -0

+{
+    int i, young = 0;
+
+    for (i = 0; i < nr; ++i, ++ptep, addr += PAGE_SIZE)
+        young |= ptep_clear_flush_young(vma, addr, ptep);
+

Why don't we use a similar loop we use in clear_young_dirty_ptes() or clear_full_ptes() etc? It's not only consistent but also optimizes out the first check for nr. for (;;) {
    young |= ptep_clear_flush_young(vma, addr, ptep);
    if (--nr == 0)
        break;
    ptep++;
    addr += PAGE_SIZE;
}

Quoted 28 lines

+    return young;

include/linux/mmu_notifier.h +3 -0

+}
+#endif
+

  /*
   * On some architectures hardware does not set page access bit when accessing
   * memory page, it is responsibility of software setting this bit. It brings

mm/rmap.c +2 -1

index e805ddc5a27b..985ab0b085ba 100644
@@ -828,9 +828,11 @@ static bool folio_referenced_one(struct folio 
 *folio,
       struct folio_referenced_arg *pra = arg;
       DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0);
       int ptes = 0, referenced = 0;
+    unsigned int nr;
       while (page_vma_mapped_walk(&pvmw)) {
           address = pvmw.address;
+        nr = 1;
           if (vma->vm_flags & VM_LOCKED) {
               ptes++;

@@ -875,9 +877,24 @@ static bool folio_referenced_one(struct folio 
 *folio,
               if (lru_gen_look_around(&pvmw))
                   referenced++;
           } else if (pvmw.pte) {
-            if (ptep_clear_flush_young_notify(vma, address,

mm/rmap.c static bool folio_referenced_one(struct folio *folio, · line 884

Quoted 6 lines

mm/rmap.c +5 -1

-                        pvmw.pte))
+            if (folio_test_large(folio)) {
+                unsigned long end_addr =
+                    pmd_addr_end(address, vma->vm_end);
+                unsigned int max_nr =
+                    (end_addr - address) >> PAGE_SHIFT;

Good news: you can both fit into a single line as we are allowed to exceed 80c if it aids readability.

mm/rmap.c static bool folio_referenced_one(struct folio *folio, · line 891

Quoted 7 lines

mm/rmap.c +7 -0

+                pte_t pteval = ptep_get(pvmw.pte);
+
+                nr = folio_pte_batch(folio, pvmw.pte,
+                             pteval, max_nr);
+            }
+
+            ptes += nr;

I'm not sure about whether we should mess with the "ptes" variable that is so far only used for VM_LOCKED vmas. See below, maybe we can just avoid that.

mm/rmap.c static bool folio_referenced_one(struct folio *folio, · line 893

Quoted 3 lines

mm/rmap.c +2 -0

+            if (clear_flush_young_ptes_notify(vma, address,
+                        pvmw.pte, nr))

Could maybe fit that into a single line as well, whatever you prefer.

Quoted 11 lines

mm/rmap.c +3 -0

                   referenced++;
+            /* Skip the batched PTEs */
+            pvmw.pte += nr - 1;
+            pvmw.address += (nr - 1) * PAGE_SIZE;
           } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
               if (pmdp_clear_flush_young_notify(vma, address,

                          pvmw.pmd))

mm/rmap.c +0 -0

@@ -887,7 +904,15 @@ static bool folio_referenced_one(struct folio 
 *folio,
               WARN_ON_ONCE(1);
           }

mm/rmap.c static bool folio_referenced_one(struct folio · line 915

Quoted 10 lines

mm/rmap.c +9 -1

-        pra->mapcount--;
+        pra->mapcount -= nr;
+        /*
+         * If we are sure that we batched the entire folio,
+         * we can just optimize and stop right here.
+         */
+        if (ptes == pvmw.nr_pages) {
+            page_vma_mapped_walk_done(&pvmw);
+            break;
+        }

Why not check for !pra->mapcount? Then you can also drop the comment, because it's exactly the same thing we check after the loop to indicate what to return to the caller.

And you will not have to mess with the "ptes" variable?

Only minor stuff.

Quoted 47 lines · David Hildenbrand (Arm)

Quoted 46 lines · Baolin Wang

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  include/linux/mmu_notifier.h |  9 +++++---- include/linux/pgtable.h      | 31 +++++++++++++++++++++++++++++++ mm/rmap.c                    | 31 ++++++++++++++++++++++++++++---

  3 files changed, 64 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +3 -2

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
       range->owner = owner;
   }
-#define ptep_clear_flush_young_notify(__vma, __address, 
 __ptep)        \
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, 
 __nr)    \
   ({                                    \
       int __young;                            \
       struct vm_area_struct *___vma = __vma;                \
       unsigned long ___address = __address;                \
-    __young = ptep_clear_flush_young(___vma, ___address, __ptep);    \
+    unsigned int ___nr = __nr;                    \
+    __young = clear_flush_young_ptes(___vma, ___address, __ptep, 
 ___nr);    \
       __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,    \

Quoted 8 lines

Quoted 6 lines

include/linux/mmu_notifier.h +1 -1

                             ___address,        \
                             ___address +        \
-                            PAGE_SIZE);    \
+                          ___nr * PAGE_SIZE);    \
       __young;                            \

  })

Man that's ugly, Not your fault, but can this possibly be turned into an inline function in a follow-up patch.

Yes, the cleanup of these macros is already in my follow-up patch set.

Quoted 7 lines

Quoted 5 lines

include/linux/mmu_notifier.h +3 -0

+#ifndef clear_flush_young_ptes
+/**
+ * clear_flush_young_ptes - Clear the access bit and perform a TLB 

flush for PTEs
+ *                that map consecutive pages of the same folio.

With clear_young_dirty_ptes() description in mind, this should probably be "Mark PTEs that map consecutive pages of the same folio as clean and flush the TLB" ?

IMO, “clean” is confusing here, as it sounds like clear the dirty bit to make the folio clean.

Quoted 23 lines

Quoted 14 lines

+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *

Quoted 8 lines

+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma, +                     unsigned long addr, pte_t *ptep,
+                     unsigned int nr)

Quoted 1 line

Two-tab alignment on second+ line like all similar functions here.

Sure.

Quoted 7 lines

include/linux/mmu_notifier.h

Quoted 6 lines

include/linux/mmu_notifier.h +6 -0

+{
+    int i, young = 0;
+
+    for (i = 0; i < nr; ++i, ++ptep, addr += PAGE_SIZE)
+        young |= ptep_clear_flush_young(vma, addr, ptep);
+

Quoted 8 lines

Why don't we use a similar loop we use in clear_young_dirty_ptes() or clear_full_ptes() etc? It's not only consistent but also optimizes out the first check for nr. for (;;) {
     young |= ptep_clear_flush_young(vma, addr, ptep);
     if (--nr == 0)
         break;
     ptep++;
     addr += PAGE_SIZE;
}

We’ve discussed this loop pattern before [1], and it seems that people prefer the ‘for (;;)’ loop. Do you have a strong preference for changing it back?

Quoted 2 lines

And you will not have to mess with the "ptes" variable?

We can't rely on pra->mapcount here, because a folio can be mapped in multiple VMAs. Even if the pra->mapcount is not zero, we can still call page_vma_mapped_walk_done() for the current VMA mapping when the entire folio is batched.

Quoted 69 lines · Baolin Wang

Quoted 47 lines · David Hildenbrand (Arm)

Quoted 46 lines · Baolin Wang

Currently, folio_referenced_one() always checks the young flag for each PTE sequentially, which is inefficient for large folios. This inefficiency is especially noticeable when reclaiming clean file-backed large folios, where folio_referenced() is observed as a significant performance hotspot.

Moreover, on Arm64 architecture, which supports contiguous PTEs, there is already an optimization to clear the young flags for PTEs within a contiguous range. However, this is not sufficient. We can extend this to perform batched operations for the entire large folio (which might exceed the contiguous range: CONT_PTE_SIZE).

Introduce a new API: clear_flush_young_ptes() to facilitate batched checking of the young flags and flushing TLB entries, thereby improving performance during large folio reclamation. And it will be overridden by the architecture that implements a more efficient batch operation in the following patches.

While we are at it, rename ptep_clear_flush_young_notify() to clear_flush_young_ptes_notify() to indicate that this is a batch operation.

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  include/linux/mmu_notifier.h |  9 +++++---- include/linux/pgtable.h      | 31 +++++++++++++++++++++++++++++++ mm/rmap.c                    | 31 ++++++++++++++++++++++++++++---

  3 files changed, 64 insertions(+), 7 deletions(-)

include/linux/mmu_notifier.h +3 -2

index d1094c2d5fb6..07a2bbaf86e9 100644
@@ -515,16 +515,17 @@ static inline void mmu_notifier_range_init_owner(
       range->owner = owner;
   }
-#define ptep_clear_flush_young_notify(__vma, __address, 
 __ptep)        \
+#define clear_flush_young_ptes_notify(__vma, __address, __ptep, 
 __nr)    \
   ({                                    \
       int __young;                            \
       struct vm_area_struct *___vma = __vma;                \
       unsigned long ___address = __address;                \
-    __young = ptep_clear_flush_young(___vma, ___address, __ptep);    \
+    unsigned int ___nr = __nr;                    \
+    __young = clear_flush_young_ptes(___vma, ___address, __ptep, 
 ___nr);    \
       __young |= mmu_notifier_clear_flush_young(___vma->vm_mm,    \

Quoted 8 lines

Quoted 6 lines

include/linux/mmu_notifier.h +1 -1

                             ___address,        \
                             ___address +        \
-                            PAGE_SIZE);    \
+                          ___nr * PAGE_SIZE);    \
       __young;                            \

  })

Man that's ugly, Not your fault, but can this possibly be turned into an inline function in a follow-up patch.

Yes, the cleanup of these macros is already in my follow-up patch set.

Quoted 7 lines

Quoted 5 lines

include/linux/mmu_notifier.h +3 -0

+#ifndef clear_flush_young_ptes
+/**
+ * clear_flush_young_ptes - Clear the access bit and perform a TLB 

flush for PTEs
+ *                that map consecutive pages of the same folio.

With clear_young_dirty_ptes() description in mind, this should probably be "Mark PTEs that map consecutive pages of the same folio as clean and flush the TLB" ?

Quoted 1 line

IMO, “clean” is confusing here, as it sounds like clear the dirty bit to make the folio clean.

"as old", sorry, I used the wrong part of the description.

Quoted 42 lines

Quoted 22 lines

Quoted 13 lines

+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries to clear access bit.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_clear_flush_young().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ. For example,
+ * some PTEs might be write-protected.
+ *

Quoted 8 lines

+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma, +                     unsigned long addr, pte_t *ptep,
+                     unsigned int nr)

Quoted 1 line

Two-tab alignment on second+ line like all similar functions here.

Sure.

Quoted 15 lines

include/linux/mmu_notifier.h

Quoted 6 lines

include/linux/mmu_notifier.h +6 -0

+{
+    int i, young = 0;
+
+    for (i = 0; i < nr; ++i, ++ptep, addr += PAGE_SIZE)
+        young |= ptep_clear_flush_young(vma, addr, ptep);
+

Why don't we use a similar loop we use in clear_young_dirty_ptes() or clear_full_ptes() etc? It's not only consistent but also optimizes out the first check for nr. for (;;) {
     young |= ptep_clear_flush_young(vma, addr, ptep);
     if (--nr == 0)
         break;
     ptep++;
     addr += PAGE_SIZE;
}

Quoted 1 line

We’ve discussed this loop pattern before [1], and it seems that people prefer the ‘for (;;)’ loop. Do you have a strong preference for changing it back?

Yes, to make all such helpers look consistent. Note that your version was also not consistent with the other variants.

Ryans point was about avoiding two ptep_clear_flush_young() calls, which the for(;;) avoids as well.

Factor out the contpte block's address and ptep alignment into a new helper, and will be reused in the following patch.

No functional changes.

Reviewed-by Ryan Roberts <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---

Currently, contpte_ptep_test_and_clear_young() and contpte_ptep_clear_flush_young() only clear the young flag and flush TLBs for PTEs within the contiguous range. To support batch PTE operations for other sized large folios in the following patches, adding a new parameter to specify the number of PTEs that map consecutive pages of the same large folio in a single VMA and a single page table.

While we are at it, rename the functions to maintain consistency with other contpte_*() functions.

Signed-off-by Baolin Wang <[email protected]>

---
 arch/arm64/include/asm/pgtable.h | 12 ++++++------
 arch/arm64/mm/contpte.c          | 33 ++++++++++++++++++--------------
 2 files changed, 25 insertions(+), 20 deletions(-)

arch/arm64/include/asm/pgtable.h +6 -6

index 445e18e92221..5e9ff16146c3 100644
@@ -1648,10 +1648,10 @@ extern void contpte_clear_full_ptes(struct mm_struct *mm, unsigned long addr,
 extern pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
 				unsigned long addr, pte_t *ptep,
 				unsigned int nr, int full);
-extern int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma,
-				unsigned long addr, pte_t *ptep);
-extern int contpte_ptep_clear_flush_young(struct vm_area_struct *vma,
-				unsigned long addr, pte_t *ptep);
+int contpte_test_and_clear_young_ptes(struct vm_area_struct *vma,
+				unsigned long addr, pte_t *ptep, unsigned int nr);
+int contpte_clear_flush_young_ptes(struct vm_area_struct *vma,
+				unsigned long addr, pte_t *ptep, unsigned int nr);
 extern void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
 				pte_t *ptep, unsigned int nr);
 extern int contpte_ptep_set_access_flags(struct vm_area_struct *vma,

@@ -1823,7 +1823,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
 	if (likely(!pte_valid_cont(orig_pte)))
 		return __ptep_test_and_clear_young(vma, addr, ptep);
 
-	return contpte_ptep_test_and_clear_young(vma, addr, ptep);
+	return contpte_test_and_clear_young_ptes(vma, addr, ptep, 1);
 }
 
 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH

@@ -1835,7 +1835,7 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
 	if (likely(!pte_valid_cont(orig_pte)))
 		return __ptep_clear_flush_young(vma, addr, ptep);
 
-	return contpte_ptep_clear_flush_young(vma, addr, ptep);
+	return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
 }
 
 #define wrprotect_ptes wrprotect_ptes

arch/arm64/mm/contpte.c +19 -14

index e4ddeb46f25d..b929a455103f 100644
@@ -508,8 +508,9 @@ pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
 }
 EXPORT_SYMBOL_GPL(contpte_get_and_clear_full_ptes);
 
-int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma,
-					unsigned long addr, pte_t *ptep)
+int contpte_test_and_clear_young_ptes(struct vm_area_struct *vma,
+					unsigned long addr, pte_t *ptep,
+					unsigned int nr)
 {
 	/*
 	 * ptep_clear_flush_young() technically requires us to clear the access

@@ -518,41 +519,45 @@ int contpte_ptep_test_and_clear_young(struct vm_area_struct *vma,
 	 * contig range when the range is covered by a single folio, we can get
 	 * away with clearing young for the whole contig range here, so we avoid
 	 * having to unfold.
+	 *
+	 * The 'nr' means consecutive (present) PTEs that map consecutive pages
+	 * of the same large folio in a single VMA and a single page table.
 	 */
 
+	unsigned long end = addr + nr * PAGE_SIZE;
 	int young = 0;
-	int i;
 
-	ptep = contpte_align_down(ptep);
-	addr = ALIGN_DOWN(addr, CONT_PTE_SIZE);
-
-	for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE)
+	ptep = contpte_align_addr_ptep(&addr, &end, ptep, nr);
+	for (; addr != end; ptep++, addr += PAGE_SIZE)
 		young |= __ptep_test_and_clear_young(vma, addr, ptep);
 
 	return young;
 }
-EXPORT_SYMBOL_GPL(contpte_ptep_test_and_clear_young);
+EXPORT_SYMBOL_GPL(contpte_test_and_clear_young_ptes);
 
-int contpte_ptep_clear_flush_young(struct vm_area_struct *vma,
-					unsigned long addr, pte_t *ptep)
+int contpte_clear_flush_young_ptes(struct vm_area_struct *vma,
+				unsigned long addr, pte_t *ptep,
+				unsigned int nr)
 {
 	int young;
 
-	young = contpte_ptep_test_and_clear_young(vma, addr, ptep);
+	young = contpte_test_and_clear_young_ptes(vma, addr, ptep, nr);
 
 	if (young) {
+		unsigned long end = addr + nr * PAGE_SIZE;
+
+		contpte_align_addr_ptep(&addr, &end, ptep, nr);
 		/*
 		 * See comment in __ptep_clear_flush_young(); same rationale for
 		 * eliding the trailing DSB applies here.
 		 */
-		addr = ALIGN_DOWN(addr, CONT_PTE_SIZE);
-		__flush_tlb_range_nosync(vma->vm_mm, addr, addr + CONT_PTE_SIZE,
+		__flush_tlb_range_nosync(vma->vm_mm, addr, end,
 					 PAGE_SIZE, true, 3);
 	}
 
 	return young;
 }
-EXPORT_SYMBOL_GPL(contpte_ptep_clear_flush_young);
+EXPORT_SYMBOL_GPL(contpte_clear_flush_young_ptes);
 
 void contpte_wrprotect_ptes(struct mm_struct *mm, unsigned long addr,
 					pte_t *ptep, unsigned int nr)

Currently, contpte_ptep_test_and_clear_young() and contpte_ptep_clear_flush_young() only clear the young flag and flush TLBs for PTEs within the contiguous range. To support batch PTE operations for other sized large folios in the following patches, adding a new parameter to specify the number of PTEs that map consecutive pages of the same large folio in a single VMA and a single page table.

While we are at it, rename the functions to maintain consistency with other contpte_*() functions.

Signed-off-by Baolin Wang <[email protected]>

---

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

arch/arm64/include/asm/pgtable.h +4 -0

+
+	return contpte_clear_flush_young_ptes(vma, addr, ptep, nr);
+}
+
 #define wrprotect_ptes wrprotect_ptes
 static __always_inline void wrprotect_ptes(struct mm_struct *mm,
 				unsigned long addr, pte_t *ptep, unsigned int nr)

Quoted 10 lines · Baolin Wang

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

Hi everyone, I ran mm-new through my AI review prompts and this one was flagged. AI review below:

Quoted 4 lines

arch/arm64/include/asm/pgtable.h static inline int ptep_clear_flush_young(struct vm_area_struct *vma, · line 1847

Quoted 10 lines

arch/arm64/include/asm/pgtable.h +7 -0

@@ -1838,6 +1838,17 @@ static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
  	return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
  }
 
+#define clear_flush_young_ptes clear_flush_young_ptes
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+					 unsigned long addr, pte_t *ptep,
+					 unsigned int nr)
+{
+	if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
+		return __ptep_clear_flush_young(vma, addr, ptep);

Should this be checking !pte_valid_cont() instead of !pte_cont()?

The existing ptep_clear_flush_young() above uses !pte_valid_cont() to determine when to take the fast path. The new function only checks !pte_cont(), which differs when handling non-present PTEs.

Non-present PTEs (device-private, device-exclusive) can reach clear_flush_young_ptes() through folio_referenced_one()-> clear_flush_young_ptes_notify(). These entries may have bit 52 set as part of their encoding, but they aren't valid contiguous mappings.

With the current check, wouldn't such entries incorrectly trigger the contpte path and potentially cause contpte_clear_flush_young_ptes() to process additional unrelated PTEs beyond the intended single entry?

Quoted 42 lines · Chris Mason

Quoted 10 lines · Baolin Wang

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

Hi everyone, I ran mm-new through my AI review prompts and this one was flagged.  AI review below:

Quoted 5 lines

arch/arm64/include/asm/pgtable.h +0 -0

@@ -1838,6 +1838,17 @@ static inline int 
 ptep_clear_flush_young(struct vm_area_struct *vma,

arch/arm64/include/asm/pgtable.h static inline int · line 1848

Quoted 10 lines

arch/arm64/include/asm/pgtable.h +7 -0

       return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
   }
 
+#define clear_flush_young_ptes clear_flush_young_ptes
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+                     unsigned long addr, pte_t *ptep,
+                     unsigned int nr)
+{
+    if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
+        return __ptep_clear_flush_young(vma, addr, ptep);

Should this be checking !pte_valid_cont() instead of !pte_cont()?

The existing ptep_clear_flush_young() above uses !pte_valid_cont() to determine when to take the fast path. The new function only checks !pte_cont(), which differs when handling non-present PTEs.

Non-present PTEs (device-private, device-exclusive) can reach clear_flush_young_ptes() through folio_referenced_one()-> clear_flush_young_ptes_notify(). These entries may have bit 52 set as part of their encoding, but they aren't valid contiguous mappings.

Quoted 3 lines

With the current check, wouldn't such entries incorrectly trigger the contpte path and potentially cause contpte_clear_flush_young_ptes() to process additional unrelated PTEs beyond the intended single entry?

Indeed. I previously discussed with Ryan whether using pte_cont() was enough, and we believed that invalid PTEs wouldn’t have the PTE_CONT bit set. But we clearly missed the device-folio cases. Thanks for reporting.

Andrew, could you please squash the following fix into this patch? If you prefer a new version, please let me know. Thanks.

Quoted 51 lines · Baolin Wang

Quoted 37 lines · Chris Mason

Quoted 10 lines · Baolin Wang

Implement the Arm64 architecture-specific clear_flush_young_ptes() to enable batched checking of young flags and TLB flushing, improving performance during large folio reclamation.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 33% performance improvement on my Arm64 32-core server (and 10%+ improvement on my X86 machine). Meanwhile, the hotspot folio_check_references() dropped from approximately 35% to around 5%.

Hi everyone, I ran mm-new through my AI review prompts and this one was flagged.  AI review below:

Quoted 5 lines

arch/arm64/include/asm/pgtable.h +0 -0

@@ -1838,6 +1838,17 @@ static inline int 
 ptep_clear_flush_young(struct vm_area_struct *vma,

arch/arm64/include/asm/pgtable.h static inline int · line 1848

Quoted 10 lines

arch/arm64/include/asm/pgtable.h +7 -0

       return contpte_clear_flush_young_ptes(vma, addr, ptep, 1);
   }
 
+#define clear_flush_young_ptes clear_flush_young_ptes
+static inline int clear_flush_young_ptes(struct vm_area_struct *vma,
+                     unsigned long addr, pte_t *ptep,
+                     unsigned int nr)
+{
+    if (likely(nr == 1 && !pte_cont(__ptep_get(ptep))))
+        return __ptep_clear_flush_young(vma, addr, ptep);

Should this be checking !pte_valid_cont() instead of !pte_cont()?

The existing ptep_clear_flush_young() above uses !pte_valid_cont() to determine when to take the fast path. The new function only checks !pte_cont(), which differs when handling non-present PTEs.

Quoted 8 lines

Non-present PTEs (device-private, device-exclusive) can reach clear_flush_young_ptes() through folio_referenced_one()-> clear_flush_young_ptes_notify(). These entries may have bit 52 set as part of their encoding, but they aren't valid contiguous mappings.

With the current check, wouldn't such entries incorrectly trigger the contpte path and potentially cause contpte_clear_flush_young_ptes() to process additional unrelated PTEs beyond the intended single entry?

Indeed. I previously discussed with Ryan whether using pte_cont() was enough, and we believed that invalid PTEs wouldn’t have the PTE_CONT bit set. But we clearly missed the device-folio cases. Thanks for reporting.

Quoted 1 line

Andrew, could you please squash the following fix into this patch? If you prefer a new version, please let me know. Thanks.

Isn't the real problem that we should never ever ever ever, try clearing the young bit on a non-present pte?

See damon_ptep_mkold() how that is handled with the flushing/notify.

There needs to be a pte_present() check in the caller.

BUT

I recall that folio_referenced() should never apply to ZONE_DEVICE folios. folio_referenced() is only called from memory reclaim code, and ZONE_DEVICE pages never get reclaimed through vmscan.c

Quoted 6 lines · David Hildenbrand (Arm)

Quoted 5 lines · Baolin Wang

Indeed. I previously discussed with Ryan whether using pte_cont() was enough, and we believed that invalid PTEs wouldn’t have the PTE_CONT bit set. But we clearly missed the device-folio cases. Thanks for reporting.

Quoted 7 lines

Quoted 1 line

Andrew, could you please squash the following fix into this patch? If you prefer a new version, please let me know. Thanks.

Isn't the real problem that we should never ever ever ever, try clearing the young bit on a non-present pte?

See damon_ptep_mkold() how that is handled with the flushing/notify.

There needs to be a pte_present() check in the caller.

The handling of ZONE_DEVICE memory in check_pte() makes me me doubt my earlier understanding. And I think you are right.

    } else if (pte_present(ptent)) {
        pfn = pte_pfn(ptent);
    } else {
        const softleaf_t entry = softleaf_from_pte(ptent);

        /* Handle un-addressable ZONE_DEVICE memory */ if (!softleaf_is_device_private(entry) && !softleaf_is_device_exclusive(entry)) return false;

        pfn = softleaf_to_pfn(entry);
    }

Quoted 3 lines

BUT

I recall that folio_referenced() should never apply to ZONE_DEVICE folios. folio_referenced() is only called from memory reclaim code, and ZONE_DEVICE pages never get reclaimed through vmscan.c

Thanks for clarifying. So I can drop the pte valid check.

Quoted 35 lines · Baolin Wang

Quoted 6 lines · David Hildenbrand (Arm)

Quoted 5 lines · Baolin Wang

Indeed. I previously discussed with Ryan whether using pte_cont() was enough, and we believed that invalid PTEs wouldn’t have the PTE_CONT bit set. But we clearly missed the device-folio cases. Thanks for reporting.

Quoted 7 lines

Quoted 1 line

Andrew, could you please squash the following fix into this patch? If you prefer a new version, please let me know. Thanks.

Isn't the real problem that we should never ever ever ever, try clearing the young bit on a non-present pte?

See damon_ptep_mkold() how that is handled with the flushing/notify.

There needs to be a pte_present() check in the caller.

The handling of ZONE_DEVICE memory in check_pte() makes me me doubt my earlier understanding. And I think you are right.

     } else if (pte_present(ptent)) {
         pfn = pte_pfn(ptent);
     } else {
         const softleaf_t entry = softleaf_from_pte(ptent);

         /* Handle un-addressable ZONE_DEVICE memory */ if (!softleaf_is_device_private(entry) && !softleaf_is_device_exclusive(entry)) return false;

         pfn = softleaf_to_pfn(entry);
     }

Quoted 2 lines

BUT

Quoted 3 lines

Quoted 1 line

I recall that folio_referenced() should never apply to ZONE_DEVICE folios. folio_referenced() is only called from memory reclaim code, and ZONE_DEVICE pages never get reclaimed through vmscan.c

Thanks for clarifying. So I can drop the pte valid check.

We should probably add a safety check in folio_referenced(), warning if we would ever get a ZONE_DEVICE folio passed.

Can someone look into that ? :)

Quoted 41 lines · Baolin Wang

Quoted 40 lines · David Hildenbrand (Arm)

Quoted 13 lines · Baolin Wang

Quoted 11 lines

Indeed. I previously discussed with Ryan whether using pte_cont() was enough, and we believed that invalid PTEs wouldn’t have the PTE_CONT bit set. But we clearly missed the device-folio cases. Thanks for reporting.

Andrew, could you please squash the following fix into this patch? If you prefer a new version, please let me know. Thanks.

Quoted 6 lines

Isn't the real problem that we should never ever ever ever, try clearing the young bit on a non-present pte?

See damon_ptep_mkold() how that is handled with the flushing/notify.

There needs to be a pte_present() check in the caller.

The handling of ZONE_DEVICE memory in check_pte() makes me me doubt my earlier understanding. And I think you are right.

     } else if (pte_present(ptent)) {
         pfn = pte_pfn(ptent);
     } else {
         const softleaf_t entry = softleaf_from_pte(ptent);

         /* Handle un-addressable ZONE_DEVICE memory */ if (!softleaf_is_device_private(entry) && !softleaf_is_device_exclusive(entry)) return false;

         pfn = softleaf_to_pfn(entry);
     }

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

BUT

I recall that folio_referenced() should never apply to ZONE_DEVICE folios. folio_referenced() is only called from memory reclaim code, and ZONE_DEVICE pages never get reclaimed through vmscan.c

Quoted 7 lines

Quoted 2 lines

Thanks for clarifying. So I can drop the pte valid check.

We should probably add a safety check in folio_referenced(), warning if we would ever get a ZONE_DEVICE folio passed.

Can someone look into that ? :)

Sure, I can take a close look and address that in my follow-up patchset.

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:
real 0m0.249s
user 0m0.000s
sys 0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
 	end_addr = pmd_addr_end(addr, vma->vm_end);
 	max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-	/* We only support lazyfree batching for now ... */
-	if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+	/* We only support lazyfree or file folios batching for now ... */
+	if (folio_test_anon(folio) && folio_test_swapbacked(folio))
 		return 1;
+
 	if (pte_unused(pte))
 		return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 			 *
 			 * See Documentation/mm/mmu_notifier.rst
 			 */
-			dec_mm_counter(mm, mm_counter_file(folio));
+			add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
 		}

discard:

		if (unlikely(folio_test_hugetlb(folio))) {

Signature · 1 line

2.47.3

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      end_addr = pmd_addr_end(addr, vma->vm_end);
      max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
              return 1;
+
      if (pte_unused(pte))
              return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                       *
                       * See Documentation/mm/mmu_notifier.rst
                       */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
              }

discard:

Quoted 4 lines

              if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

    try_to_unmap_one()
        while (page_vma_mapped_walk(&pvmw)) {
            nr_pages = folio_unmap_pte_batch()

            if (nr_pages = folio_nr_pages(folio))
                goto walk_done;
        }

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

--
Wei Yang
Help you, Help me

Quoted 79 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      end_addr = pmd_addr_end(addr, vma->vm_end);
      max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
              return 1;
+
      if (pte_unused(pte))
              return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                       *
                       * See Documentation/mm/mmu_notifier.rst
                       */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
              }

discard:

Quoted 4 lines

              if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

    try_to_unmap_one()
        while (page_vma_mapped_walk(&pvmw)) {
            nr_pages = folio_unmap_pte_batch()

            if (nr_pages = folio_nr_pages(folio))
                goto walk_done;
        }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

               do {
                       pvmw->address += PAGE_SIZE;
                       if (pvmw->address >= end)
                               return not_found(pvmw);
                       /* Did we cross page table boundary? */
                       if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                               if (pvmw->ptl) {
                                       spin_unlock(pvmw->ptl);
                                       pvmw->ptl = NULL;

                               }
                               pte_unmap(pvmw->pte);
                               pvmw->pte = NULL;
                               pvmw->flags |= PVMW_PGTABLE_CROSSED;
                               goto restart;
                       }
                       pvmw->pte++;
               } while (pte_none(ptep_get(pvmw->pte)));

Quoted 104 lines · Barry Song

Quoted 79 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      end_addr = pmd_addr_end(addr, vma->vm_end);
      max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
              return 1;
+
      if (pte_unused(pte))
              return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                       *
                       * See Documentation/mm/mmu_notifier.rst
                       */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
              }

discard:

Quoted 4 lines

              if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

    try_to_unmap_one()
        while (page_vma_mapped_walk(&pvmw)) {
            nr_pages = folio_unmap_pte_batch()

            if (nr_pages = folio_nr_pages(folio))
                goto walk_done;
        }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

               do {
                       pvmw->address += PAGE_SIZE;
                       if (pvmw->address >= end)
                               return not_found(pvmw);
                       /* Did we cross page table boundary? */
                       if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                               if (pvmw->ptl) {
                                       spin_unlock(pvmw->ptl);
                                       pvmw->ptl = NULL;

Quoted 9 lines

                               }
                               pte_unmap(pvmw->pte);
                               pvmw->pte = NULL;
                               pvmw->flags |= PVMW_PGTABLE_CROSSED;
                               goto restart;
                       }
                       pvmw->pte++;
               } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +0 -0

index 9e5bd4834481..ea1afec7c802 100644

Not sure this is reasonable.

Quoted 133 lines · Wei Yang

Quoted 104 lines · Barry Song

Quoted 79 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
       end_addr = pmd_addr_end(addr, vma->vm_end);
       max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
               return 1;
+
       if (pte_unused(pte))
               return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                        *
                        * See Documentation/mm/mmu_notifier.rst
                        */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
               }

discard:

Quoted 4 lines

               if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

     try_to_unmap_one()
         while (page_vma_mapped_walk(&pvmw)) {
             nr_pages = folio_unmap_pte_batch()

             if (nr_pages = folio_nr_pages(folio))
                 goto walk_done;
         }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

                do {
                        pvmw->address += PAGE_SIZE;
                        if (pvmw->address >= end)
                                return not_found(pvmw);
                        /* Did we cross page table boundary? */
                        if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                                if (pvmw->ptl) {
                                        spin_unlock(pvmw->ptl);
                                        pvmw->ptl = NULL;

Quoted 9 lines

                                }
                                pte_unmap(pvmw->pte);
                                pvmw->pte = NULL;
                                pvmw->flags |= PVMW_PGTABLE_CROSSED;
                                goto restart;
                        }
                        pvmw->pte++;
                } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +4 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                  */
                 if (nr_pages == folio_nr_pages(folio))
                         goto walk_done;
+               else {
+                       pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                       pvmw.pte += nr_pages - 1;
+               }

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 3 lines

mm/rmap.c +0 -0

                 continue;
  walk_abort:
                 ret = false;

I feel this couples the PTE walk iteration with the unmap operation, which does not seem fine to me. It also appears to affect only corner cases.

Quoted 134 lines · Barry Song

Quoted 133 lines · Wei Yang

Quoted 104 lines · Barry Song

Quoted 79 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
       end_addr = pmd_addr_end(addr, vma->vm_end);
       max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
               return 1;
+
       if (pte_unused(pte))
               return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                        *
                        * See Documentation/mm/mmu_notifier.rst
                        */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
               }

discard:

Quoted 4 lines

               if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

     try_to_unmap_one()
         while (page_vma_mapped_walk(&pvmw)) {
             nr_pages = folio_unmap_pte_batch()

             if (nr_pages = folio_nr_pages(folio))
                 goto walk_done;
         }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

                do {
                        pvmw->address += PAGE_SIZE;
                        if (pvmw->address >= end)
                                return not_found(pvmw);
                        /* Did we cross page table boundary? */
                        if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                                if (pvmw->ptl) {
                                        spin_unlock(pvmw->ptl);
                                        pvmw->ptl = NULL;

Quoted 9 lines

                                }
                                pte_unmap(pvmw->pte);
                                pvmw->pte = NULL;
                                pvmw->flags |= PVMW_PGTABLE_CROSSED;
                                goto restart;
                        }
                        pvmw->pte++;
                } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +4 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                  */
                 if (nr_pages == folio_nr_pages(folio))
                         goto walk_done;
+               else {
+                       pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                       pvmw.pte += nr_pages - 1;
+               }

Quoted 8 lines

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 3 lines

mm/rmap.c +0 -0

                 continue;
  walk_abort:
                 ret = false;

I feel this couples the PTE walk iteration with the unmap operation, which does not seem fine to me. It also appears to affect only corner cases.

Agree. There may be no performance gains, so I also prefer to leave it as is.

Quoted 101 lines · Barry Song

Quoted 77 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      end_addr = pmd_addr_end(addr, vma->vm_end);
      max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
              return 1;
+
      if (pte_unused(pte))
              return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                       *
                       * See Documentation/mm/mmu_notifier.rst
                       */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
              }

discard:

Quoted 4 lines

              if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

    try_to_unmap_one()
        while (page_vma_mapped_walk(&pvmw)) {
            nr_pages = folio_unmap_pte_batch()

            if (nr_pages = folio_nr_pages(folio))
                goto walk_done;
        }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

               do {
                       pvmw->address += PAGE_SIZE;
                       if (pvmw->address >= end)
                               return not_found(pvmw);
                       /* Did we cross page table boundary? */
                       if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                               if (pvmw->ptl) {
                                       spin_unlock(pvmw->ptl);
                                       pvmw->ptl = NULL;

Quoted 9 lines

                               }
                               pte_unmap(pvmw->pte);
                               pvmw->pte = NULL;
                               pvmw->flags |= PVMW_PGTABLE_CROSSED;
                               goto restart;
                       }
                       pvmw->pte++;
               } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +0 -0

index 9e5bd4834481..ea1afec7c802 100644

Not sure this is reasonable.

I am of the opinion that we should do something like this. In the internal pvmw code, we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none to not none, and we will lose the batching effect. I also plan to extend support to anonymous folios (therefore generalizing for all types of memory) which will set a batch of ptes as swap, and the internal pvmw code won't be able to skip through the batch.

[1] https://lore.kernel.org/linux-mm/[email protected]/

Quoted 125 lines · Wei Yang

Quoted 101 lines · Barry Song

Quoted 77 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      end_addr = pmd_addr_end(addr, vma->vm_end);
      max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
              return 1;
+
      if (pte_unused(pte))
              return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                       *
                       * See Documentation/mm/mmu_notifier.rst
                       */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
              }

discard:

Quoted 4 lines

              if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

    try_to_unmap_one()
        while (page_vma_mapped_walk(&pvmw)) {
            nr_pages = folio_unmap_pte_batch()

            if (nr_pages = folio_nr_pages(folio))
                goto walk_done;
        }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

               do {
                       pvmw->address += PAGE_SIZE;
                       if (pvmw->address >= end)
                               return not_found(pvmw);
                       /* Did we cross page table boundary? */
                       if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                               if (pvmw->ptl) {
                                       spin_unlock(pvmw->ptl);
                                       pvmw->ptl = NULL;

Quoted 9 lines

                               }
                               pte_unmap(pvmw->pte);
                               pvmw->pte = NULL;
                               pvmw->flags |= PVMW_PGTABLE_CROSSED;
                               goto restart;
                       }
                       pvmw->pte++;
               } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +1 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
               */
              if (nr_pages == folio_nr_pages(folio))
                      goto walk_done;
+             else {

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 6 lines

mm/rmap.c +3 -0

+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }
              continue;
 walk_abort:
              ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code,

we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none to not none, and we will lose the batching effect. I also plan to extend support to anonymous folios (therefore generalizing for all types of memory) which will set a batch of ptes as swap, and the internal pvmw code won't be able to skip through the batch.

Thanks for catching this, Dev. I already filter out some of the more complex cases, for example:
if (pte_unused(pte))

        return 1;

Since the userfaultfd write-protection case is also a corner case, could we filter it out as well?

mm/rmap.c +3 -0

index c86f1135222b..6bb8ba6f046e 100644
@@ -1870,6 +1870,9 @@ static inline unsigned int
 folio_unmap_pte_batch(struct folio *folio,
        if (pte_unused(pte))
                return 1;
 
+       if (userfaultfd_wp(vma))
+               return 1;
+
        return folio_pte_batch(folio, pvmw->pte, pte, max_nr);
 }

Just offering a second option — yours is probably better.

Quoted 128 lines · Dev Jain

Quoted 125 lines · Wei Yang

Quoted 101 lines · Barry Song

Quoted 77 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
       end_addr = pmd_addr_end(addr, vma->vm_end);
       max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
               return 1;
+
       if (pte_unused(pte))
               return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                        *
                        * See Documentation/mm/mmu_notifier.rst
                        */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
               }

discard:

Quoted 4 lines

               if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

     try_to_unmap_one()
         while (page_vma_mapped_walk(&pvmw)) {
             nr_pages = folio_unmap_pte_batch()

             if (nr_pages = folio_nr_pages(folio))
                 goto walk_done;
         }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

                do {
                        pvmw->address += PAGE_SIZE;
                        if (pvmw->address >= end)
                                return not_found(pvmw);
                        /* Did we cross page table boundary? */
                        if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                                if (pvmw->ptl) {
                                        spin_unlock(pvmw->ptl);
                                        pvmw->ptl = NULL;

Quoted 9 lines

                                }
                                pte_unmap(pvmw->pte);
                                pvmw->pte = NULL;
                                pvmw->flags |= PVMW_PGTABLE_CROSSED;
                                goto restart;
                        }
                        pvmw->pte++;
                } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +1 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                */
               if (nr_pages == folio_nr_pages(folio))
                       goto walk_done;
+             else {

Quoted 8 lines

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 6 lines

mm/rmap.c +3 -0

+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }
               continue;
  walk_abort:
               ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code,

I am still not convinced that skipping PTEs in try_to_unmap_one() is the right place. If we really want to skip certain PTEs early, should we instead hint page_vma_mapped_walk()? That said, I don't see much value in doing so, since in most cases nr is either 1 or folio_nr_pages(folio).

Quoted 6 lines

we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none to not none, and we will lose the batching effect. I also plan to extend support to anonymous folios (therefore generalizing for all types of memory) which will set a batch of ptes as swap, and the internal pvmw code won't be able to skip through the batch.

Thanks for catching this, Dev. I already filter out some of the more complex cases, for example:
if (pte_unused(pte))

         return 1;

Since the userfaultfd write-protection case is also a corner case, could we filter it out as well?

mm/rmap.c +0 -0

index c86f1135222b..6bb8ba6f046e 100644

Quoted 118 lines · Dev Jain

Quoted 116 lines · Wei Yang

Quoted 101 lines · Barry Song

Quoted 77 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      end_addr = pmd_addr_end(addr, vma->vm_end);
      max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
              return 1;
+
      if (pte_unused(pte))
              return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                       *
                       * See Documentation/mm/mmu_notifier.rst
                       */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
              }

discard:

Quoted 4 lines

              if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

    try_to_unmap_one()
        while (page_vma_mapped_walk(&pvmw)) {
            nr_pages = folio_unmap_pte_batch()

            if (nr_pages = folio_nr_pages(folio))
                goto walk_done;
        }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

               do {
                       pvmw->address += PAGE_SIZE;
                       if (pvmw->address >= end)
                               return not_found(pvmw);
                       /* Did we cross page table boundary? */
                       if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                               if (pvmw->ptl) {
                                       spin_unlock(pvmw->ptl);
                                       pvmw->ptl = NULL;

Quoted 9 lines

                               }
                               pte_unmap(pvmw->pte);
                               pvmw->pte = NULL;
                               pvmw->flags |= PVMW_PGTABLE_CROSSED;
                               goto restart;
                       }
                       pvmw->pte++;
               } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

Quoted 16 lines

Quoted 5 lines

mm/rmap.c +0 -0

index 9e5bd4834481..ea1afec7c802 100644

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 10 lines

mm/rmap.c +4 -0

@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
               */
              if (nr_pages == folio_nr_pages(folio))
                      goto walk_done;
+             else {
+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }
              continue;
 walk_abort:
              ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code,

I am still not convinced that skipping PTEs in try_to_unmap_one() is the right place. If we really want to skip certain PTEs early, should we instead hint page_vma_mapped_walk()? That said, I don't see much value in doing so, since in most cases nr is either 1 or folio_nr_pages(folio).

Quoted 6 lines

we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none to not none, and we will lose the batching effect. I also plan to extend support to anonymous folios (therefore generalizing for all types of memory) which will set a batch of ptes as swap, and the internal pvmw code won't be able to skip through the batch.

Thanks for catching this, Dev. I already filter out some of the more complex cases, for example:
if (pte_unused(pte))

        return 1;

Since the userfaultfd write-protection case is also a corner case, could we filter it out as well?

mm/rmap.c +3 -0

index c86f1135222b..6bb8ba6f046e 100644
@@ -1870,6 +1870,9 @@ static inline unsigned int
 folio_unmap_pte_batch(struct folio *folio,
        if (pte_unused(pte))
                return 1;
 
+       if (userfaultfd_wp(vma))
+               return 1;
+
        return folio_pte_batch(folio, pvmw->pte, pte, max_nr);
 }

Just offering a second option — yours is probably better.

Thanks
Barry

Quoted 170 lines · Barry Song

Quoted 118 lines · Dev Jain

Quoted 116 lines · Wei Yang

Quoted 101 lines · Barry Song

Quoted 77 lines · Wei Yang

Quoted 58 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:

real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -3

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
       end_addr = pmd_addr_end(addr, vma->vm_end);
       max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now ... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
               return 1;
+
       if (pte_unused(pte))
               return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                        *
                        * See Documentation/mm/mmu_notifier.rst
                        */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
               }

discard:

Quoted 4 lines

               if (unlikely(folio_test_hugetlb(folio))) {

--
2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

     try_to_unmap_one()
         while (page_vma_mapped_walk(&pvmw)) {
             nr_pages = folio_unmap_pte_batch()

             if (nr_pages = folio_nr_pages(folio))
                 goto walk_done;
         }

Quoted 8 lines

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from (pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to (pvmw->address + nr_pages * PAGE_SIZE), right?

Not sure my understanding is correct, if so do we have some reason not to skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:

                do {
                        pvmw->address += PAGE_SIZE;
                        if (pvmw->address >= end)
                                return not_found(pvmw);
                        /* Did we cross page table boundary? */
                        if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                                if (pvmw->ptl) {
                                        spin_unlock(pvmw->ptl);
                                        pvmw->ptl = NULL;

Quoted 9 lines

                                }
                                pte_unmap(pvmw->pte);
                                pvmw->pte = NULL;
                                pvmw->flags |= PVMW_PGTABLE_CROSSED;
                                goto restart;
                        }
                        pvmw->pte++;
                } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

Quoted 16 lines

Quoted 5 lines

mm/rmap.c +0 -0

index 9e5bd4834481..ea1afec7c802 100644

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 10 lines

mm/rmap.c +4 -0

@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                */
               if (nr_pages == folio_nr_pages(folio))
                       goto walk_done;
+             else {
+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }
               continue;
  walk_abort:
               ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code,

I am still not convinced that skipping PTEs in try_to_unmap_one() is the right place. If we really want to skip certain PTEs early, should we instead hint page_vma_mapped_walk()? That said, I don't see much value in doing so, since in most cases nr is either 1 or folio_nr_pages(folio).

Quoted 6 lines

we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none to not none, and we will lose the batching effect. I also plan to extend support to anonymous folios (therefore generalizing for all types of memory) which will set a batch of ptes as swap, and the internal pvmw code won't be able to skip through the batch.

Thanks for catching this, Dev. I already filter out some of the more complex cases, for example:
if (pte_unused(pte))

         return 1;

Since the userfaultfd write-protection case is also a corner case, could we filter it out as well?

mm/rmap.c +3 -0

index c86f1135222b..6bb8ba6f046e 100644
@@ -1870,6 +1870,9 @@ static inline unsigned int
 folio_unmap_pte_batch(struct folio *folio,
         if (pte_unused(pte))
                 return 1;
 
+       if (userfaultfd_wp(vma))
+               return 1;
+
         return folio_pte_batch(folio, pvmw->pte, pte, max_nr);
 }

Quoted 1 line

Just offering a second option — yours is probably better.

No. This is not an edge case. This is a case which gets exposed by your work, and I believe that if you intend to get the file folio batching thingy in, then you need to fix the uffd stuff too.

Quoted 199 lines · Dev Jain

Quoted 197 lines · Barry Song

Quoted 148 lines · Dev Jain

Quoted 146 lines · Wei Yang

Quoted 120 lines · Barry Song

Quoted 98 lines · Wei Yang

Quoted 70 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file
large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large
folios[1] and did not further optimize anonymous large folios or file-backed
large folios at that stage. As for file-backed large folios, the batched
unmapping support is relatively straightforward, as we only need to clear
the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to
reclaim 8G file-backed folios via the memory.reclaim interface. I can observe
75% performance improvement on my Arm64 32-core server (and 50%+ improvement
on my X86 machine) with this patch.

W/o patch:
real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:
real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1]
https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +1 -2

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int
 folio_unmap_pte_batch(struct folio *folio,
        end_addr = pmd_addr_end(addr, vma->vm_end);
        max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now

... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio)) return 1;
+
       if (pte_unused(pte))
               return 1;

mm/rmap.c +1 -2

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio
 *folio, struct vm_area_struct *vma,
                         *
                         * See Documentation/mm/mmu_notifier.rst
                         */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio),
-nr_pages);
                }

Quoted 5 lines

discard:
               if (unlikely(folio_test_hugetlb(folio))) {

Signature · 1 line

2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

     try_to_unmap_one()
         while (page_vma_mapped_walk(&pvmw)) {
             nr_pages = folio_unmap_pte_batch()

             if (nr_pages = folio_nr_pages(folio)) goto walk_done;
         }

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from
(pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to
(pvmw->address + nr_pages * PAGE_SIZE), right?

Quoted 3 lines

Not sure my understanding is correct, if so do we have some reason not to
skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:
                do {
                        pvmw->address += PAGE_SIZE;
                        if (pvmw->address >= end)
                                return not_found(pvmw); /* Did we cross page table boundary? */ if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) {
                                if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL;

Quoted 9 lines

                                }
                                pte_unmap(pvmw->pte); pvmw->pte = NULL; pvmw->flags |= PVMW_PGTABLE_CROSSED; goto restart;
                        }
                        pvmw->pte++;
                } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they
will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +1 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio
 *folio, struct vm_area_struct *vma,
                 */
                if (nr_pages == folio_nr_pages(folio))
                        goto walk_done;
+             else {

Quoted 8 lines

Quoted 6 lines

mm/rmap.c +3 -0

+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }
                continue;

  walk_abort:
               ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code,

I am still not convinced that skipping PTEs in try_to_unmap_one() is the right place. If we really want to skip certain PTEs early, should we instead hint page_vma_mapped_walk()? That said, I don't see much value in doing so, since in most cases nr is either 1 or folio_nr_pages(folio).

Quoted 11 lines

we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old
ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none
to not none, and we will lose the batching effect. I also plan to extend support to
anonymous folios (therefore generalizing for all types of memory) which will set a
batch of ptes as swap, and the internal pvmw code won't be able to skip through the
batch.

Thanks for catching this, Dev. I already filter out some of the more complex cases, for example:
if (pte_unused(pte))
         return 1;

Since the userfaultfd write-protection case is also a corner case, could we filter it out as well?

mm/rmap.c +3 -0

index c86f1135222b..6bb8ba6f046e 100644
@@ -1870,6 +1870,9 @@ static inline unsigned int
 folio_unmap_pte_batch(struct folio *folio,
          if (pte_unused(pte))
                  return 1;
 
+       if (userfaultfd_wp(vma))
+               return 1;
+
          return folio_pte_batch(folio, pvmw->pte, pte, max_nr);
 }

Quoted 7 lines

Quoted 1 line

Just offering a second option — yours is probably better.

No. This is not an edge case. This is a case which gets exposed by your work, and
I believe that if you intend to get the file folio batching thingy in, then you
need to fix the uffd stuff too.

Barry’s point isn’t that this is an edge case. I think he means that uffd is not a common performance-sensitive scenario in production. Also, we typically fall back to per-page handling for uffd cases (see finish_fault() and alloc_anon_folio()). So I perfer to follow Barry’s suggestion and filter out the uffd cases until we have test case to show performance improvement.

I also think you can continue iterating your patch[1] to support batched unmapping for uffd VMAs, and provide data to evaluate its value.

[1]
https://lore.kernel.org/linux-mm/[email protected]/

Quoted 210 lines · Baolin Wang

Quoted 199 lines · Dev Jain

Quoted 197 lines · Barry Song

Quoted 148 lines · Dev Jain

Quoted 146 lines · Wei Yang

Quoted 120 lines · Barry Song

Quoted 98 lines · Wei Yang

Quoted 70 lines · Baolin Wang

Similar to folio_referenced_one(), we can apply batched unmapping for file
large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large
folios[1] and did not further optimize anonymous large folios or file-backed
large folios at that stage. As for file-backed large folios, the batched
unmapping support is relatively straightforward, as we only need to clear
the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to
reclaim 8G file-backed folios via the memory.reclaim interface. I can observe
75% performance improvement on my Arm64 32-core server (and 50%+ improvement
on my X86 machine) with this patch.

W/o patch:
real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:
real   0m0.249s
user   0m0.000s
sys    0m0.249s

[1]
https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
mm/rmap.c | 7 ++++---

1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +1 -2

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int
 folio_unmap_pte_batch(struct folio *folio,
        end_addr = pmd_addr_end(addr, vma->vm_end);
        max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-      /* We only support lazyfree batching for now ... */
-      if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+      /* We only support lazyfree or file folios batching for now

... */
+      if (folio_test_anon(folio) && folio_test_swapbacked(folio))
               return 1;

mm/rmap.c +2 -2

+
        if (pte_unused(pte))
                return 1;

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio
 *folio, struct vm_area_struct *vma,
                         *
                         * See Documentation/mm/mmu_notifier.rst
                         */
-                      dec_mm_counter(mm, mm_counter_file(folio));
+                      add_mm_counter(mm, mm_counter_file(folio),
-nr_pages);
                }

Quoted 5 lines

discard:
               if (unlikely(folio_test_hugetlb(folio))) {

Signature · 1 line

2.47.3

Hi, Baolin

When reading your patch, I come up one small question.

Current try_to_unmap_one() has following structure:

     try_to_unmap_one()
         while (page_vma_mapped_walk(&pvmw)) {
             nr_pages = folio_unmap_pte_batch()

             if (nr_pages = folio_nr_pages(folio)) goto walk_done;
         }

I am thinking what if nr_pages > 1 but nr_pages != folio_nr_pages().

If my understanding is correct, page_vma_mapped_walk() would start from
(pvmw->address + PAGE_SIZE) in next iteration, but we have already cleared to
(pvmw->address + nr_pages * PAGE_SIZE), right?

Quoted 3 lines

Not sure my understanding is correct, if so do we have some reason not to
skip the cleared range?

I don’t quite understand your question. For nr_pages > 1 but not equal to nr_pages, page_vma_mapped_walk will skip the nr_pages - 1 PTEs inside.

take a look:

next_pte:
                do {
                        pvmw->address += PAGE_SIZE;
                        if (pvmw->address >= end)
                                return not_found(pvmw); /* Did we cross page table boundary? */ if ((pvmw->address & (PMD_SIZE - PAGE_SIZE))
== 0) {
                                if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL;

Quoted 9 lines

                                }
                                pte_unmap(pvmw->pte); pvmw->pte = NULL; pvmw->flags |= PVMW_PGTABLE_CROSSED; goto restart;
                        }
                        pvmw->pte++;
                } while (pte_none(ptep_get(pvmw->pte)));

Yes, we do it in page_vma_mapped_walk() now. Since they are pte_none(), they
will be skipped.

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +1 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio
 *folio, struct vm_area_struct *vma,
                 */
                if (nr_pages == folio_nr_pages(folio))
                        goto walk_done;
+             else {

Quoted 8 lines

Quoted 6 lines

mm/rmap.c +3 -0

+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }
                continue;
   walk_abort:

               ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code,

I am still not convinced that skipping PTEs in try_to_unmap_one() is the right place. If we really want to skip certain PTEs early, should we instead hint page_vma_mapped_walk()? That said, I don't see much value in doing so, since in most cases nr is either 1 or folio_nr_pages(folio).

Quoted 11 lines

we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old
ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none
to not none, and we will lose the batching effect. I also plan to extend support to
anonymous folios (therefore generalizing for all types of memory) which will set a
batch of ptes as swap, and the internal pvmw code won't be able to skip through the
batch.

Thanks for catching this, Dev. I already filter out some of the more complex cases, for example:
if (pte_unused(pte))
         return 1;

Since the userfaultfd write-protection case is also a corner case, could we filter it out as well?

mm/rmap.c +3 -0

index c86f1135222b..6bb8ba6f046e 100644
@@ -1870,6 +1870,9 @@ static inline unsigned int
 folio_unmap_pte_batch(struct folio *folio,
          if (pte_unused(pte))
                  return 1;
 
+       if (userfaultfd_wp(vma))
+               return 1;
+
          return folio_pte_batch(folio, pvmw->pte, pte, max_nr);
 }

Quoted 7 lines

Quoted 1 line

Just offering a second option — yours is probably better.

No. This is not an edge case. This is a case which gets exposed by your work, and
I believe that if you intend to get the file folio batching thingy in, then you
need to fix the uffd stuff too.

Quoted 6 lines

Barry’s point isn’t that this is an edge case. I think he means that uffd is not a common performance-sensitive scenario in production. Also, we typically fall back to per-page handling for uffd cases (see finish_fault() and alloc_anon_folio()). So I perfer to follow Barry’s suggestion and filter out the uffd cases until we have test case to show performance improvement.

I am of the opinion that you are making the wrong analogy here. The per-page fault fidelity is *required* for uffd.

When you say you want to support file folio batched unmapping, I think it's inappropriate to say "let us refuse to

batch if the pte mapping the file folio is smeared with a particular bit and consider it a totally different case". Instead

of getting in folio (all memory types) batched unmapping in, we have already broken this to "lazyfree folio", then

"file folio", the remaining being "anon folio". Now you intend to break "file folio" to "file folio non uffd" and "file folio uffd".

Quoted 16 lines

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +4 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
               */
              if (nr_pages == folio_nr_pages(folio))
                      goto walk_done;
+             else {
+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }
              continue;

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 2 lines

mm/rmap.c +0 -0

 walk_abort:
              ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code, we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none to not none, and we will lose the batching effect. I also plan to extend support to anonymous folios (therefore generalizing for all types of memory) which will set a

batch of ptes as swap, and the internal pvmw code won't be able to skip through the batch.

Quoted 14 lines

Quoted 14 lines

I mean maybe we can skip it in try_to_unmap_one(), for example:

mm/rmap.c +4 -0

index 9e5bd4834481..ea1afec7c802 100644
@@ -2250,6 +2250,10 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
               */
              if (nr_pages == folio_nr_pages(folio))
                      goto walk_done;
+             else {
+                     pvmw.address += PAGE_SIZE * (nr_pages - 1);
+                     pvmw.pte += nr_pages - 1;
+             }

Quoted 8 lines

mm/rmap.c static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, · line 2259

Quoted 3 lines

mm/rmap.c +0 -0

              continue;
 walk_abort:
              ret = false;

I am of the opinion that we should do something like this. In the internal pvmw code, we keep skipping ptes till the ptes are none. With my proposed uffd-fix [1], if the old ptes were uffd-wp armed, pte_install_uffd_wp_if_needed will convert all ptes from none to not none, and we will lose the batching effect. I also plan to extend support to anonymous folios (therefore generalizing for all types of memory) which will set a

I posted an RFC on anon folios quite some time ago [1]. It’s great to hear that you’re interested in taking this over.

[1] https://lore.kernel.org/all/[email protected]/

Quoted 3 lines

batch of ptes as swap, and the internal pvmw code won't be able to skip through the batch.

Interesting — I didn’t catch this issue in the RFC earlier. Back then, we only supported nr == 1 and nr == folio_nr_pages(folio). When nr == nr_pages, page_vma_mapped_walk() would break entirely. With Lance’s commit ddd05742b45b08, arbitrary nr in [1, nr_pages] is now supported, which means we have to handle all the complexity. :-)

Thanks
Barry

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:
real 0m0.249s
user 0m0.000s
sys 0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---

Similar to folio_referenced_one(), we can apply batched unmapping for file large folios to optimize the performance of file folios reclamation.

Barry previously implemented batched unmapping for lazyfree anonymous large folios[1] and did not further optimize anonymous large folios or file-backed large folios at that stage. As for file-backed large folios, the batched unmapping support is relatively straightforward, as we only need to clear the consecutive (present) PTE entries for file-backed large folios.

Performance testing:
Allocate 10G clean file-backed folios by mmap() in a memory cgroup, and try to reclaim 8G file-backed folios via the memory.reclaim interface. I can observe 75% performance improvement on my Arm64 32-core server (and 50%+ improvement on my X86 machine) with this patch.

W/o patch:

real    0m1.018s
user    0m0.000s
sys     0m1.018s

W/ patch:
real 0m0.249s
user 0m0.000s
sys 0m0.249s

[1] https://lore.kernel.org/all/[email protected]/T/#u

Reviewed-by Ryan Roberts <[email protected]>

Acked-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
 mm/rmap.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

mm/rmap.c +4 -4

index 985ab0b085ba..e1d16003c514 100644
@@ -1863,9 +1863,10 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
 	end_addr = pmd_addr_end(addr, vma->vm_end);
 	max_nr = (end_addr - addr) >> PAGE_SHIFT;
 
-	/* We only support lazyfree batching for now ... */
-	if (!folio_test_anon(folio) || folio_test_swapbacked(folio))
+	/* We only support lazyfree or file folios batching for now ... */
+	if (folio_test_anon(folio) && folio_test_swapbacked(folio))
 		return 1;
+
 	if (pte_unused(pte))
 		return 1;
 

@@ -2231,7 +2232,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 			 *
 			 * See Documentation/mm/mmu_notifier.rst
 			 */
-			dec_mm_counter(mm, mm_counter_file(folio));
+			add_mm_counter(mm, mm_counter_file(folio), -nr_pages);
 		}
 discard:
 		if (unlikely(folio_test_hugetlb(folio))) {
--

2.47.3

As Dev reported[1], it's not ready to support batched unmapping for uffd case. Let's still fallback to per-page unmapping for the uffd case.

[1] https://lore.kernel.org/linux-mm/[email protected]/

Reported-by Dev Jain <[email protected]>

Suggested-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  mm/rmap.c | 3 +++
  1 file changed, 3 insertions(+)

mm/rmap.c +0 -0

index f13480cb9f2e..172643092dcf 100644

Quoted 16 lines · Baolin Wang

As Dev reported[1], it's not ready to support batched unmapping for uffd case. Let's still fallback to per-page unmapping for the uffd case.

[1] https://lore.kernel.org/linux-mm/[email protected]/

Reported-by Dev Jain <[email protected]>

Suggested-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  mm/rmap.c | 3 +++
  1 file changed, 3 insertions(+)

mm/rmap.c +0 -0

index f13480cb9f2e..172643092dcf 100644

mm/rmap.c static inline unsigned int folio_unmap_pte_batch(struct folio *folio, · line 1958

Quoted 6 lines

mm/rmap.c +3 -0

@@ -1953,6 +1953,9 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      if (pte_unused(pte))
              return 1;
 
+     if (userfaultfd_wp(vma))
+             return 1;
+

Interesting. I was just wondering why we didn't run into that with lazyfree folios.

Staring at pte_install_uffd_wp_if_needed(), we never set the marker for anonymous VMAs.

So, yeah, if one sets lazyfree on a uffd-wp PTE, the uffd-wp bit will just get zapped alongside. Just like MADV_DONTNEED.

I'm fine with that temporary fix. But I guess the non-hacky way to handle this would be:

From 53d016d6e6f624425dbdbc2fb1dec7c91fbef15c Mon Sep 17 00:00:00 2001
From: "David Hildenbrand (Arm)" <[email protected]>
Date: Mon, 9 Feb 2026 10:52:59 +0100
Subject: [PATCH] tmp

Signed-off-by David Hildenbrand (Arm) <[email protected]>

---
  include/linux/mm_inline.h | 15 ++++++---------
  mm/memory.c               | 21 +--------------------
  mm/rmap.c                 |  2 +-
  3 files changed, 8 insertions(+), 30 deletions(-)

include/linux/mm_inline.h +6 -9

index fa2d6ba811b5..8a9a2c5f5ee3 100644
@@ -566,9 +566,8 @@ static inline pte_marker copy_pte_marker(
   *
   * Returns true if an uffd-wp pte was installed, false otherwise.
   */
-static inline bool
-pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
-                             pte_t *pte, pte_t pteval)
+static inline bool install_uffd_wp_ptes_if_needed(struct vm_area_struct *vma,
+               unsigned long addr, pte_t *pte, unsigned int nr, pte_t pteval)
  {
        bool arm_uffd_pte = false;
 

@@ -598,13 +597,11 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
        if (unlikely(pte_swp_uffd_wp_any(pteval)))
                arm_uffd_pte = true;
 
-       if (unlikely(arm_uffd_pte)) {
-               set_pte_at(vma->vm_mm, addr, pte,
-                          make_pte_marker(PTE_MARKER_UFFD_WP));
-               return true;
-       }
+       if (likely(!arm_uffd_pte))
+               return false;
 
-       return false;
+       set_ptes(vma->vm_mm, addr, pte, make_pte_marker(PTE_MARKER_UFFD_WP), nr);
+       return true;
  }
 
  static inline bool vma_has_recency(const struct vm_area_struct *vma)

mm/memory.c +1 -20

index da360a6eb8a4..0a87d02a9a69 100644
@@ -1592,29 +1592,10 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
                              unsigned long addr, pte_t *pte, int nr,
                              struct zap_details *details, pte_t pteval)
  {
-       bool was_installed = false;
-
-       if (!uffd_supports_wp_marker())
-               return false;
-
-       /* Zap on anonymous always means dropping everything */
-       if (vma_is_anonymous(vma))
-               return false;
-
        if (zap_drop_markers(details))
                return false;
 
-       for (;;) {
-               /* the PFN in the PTE is irrelevant. */
-               if (pte_install_uffd_wp_if_needed(vma, addr, pte, pteval))
-                       was_installed = true;
-               if (--nr == 0)
-                       break;
-               pte++;
-               addr += PAGE_SIZE;
-       }
-
-       return was_installed;
+       return install_uffd_wp_ptes_if_needed(vma, addr, pte, nr, pteval);
  }
 
  static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,

mm/rmap.c +1 -2

index 7b9879ef442d..f71aacf35925 100644
@@ -2061,7 +2061,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                 * we may want to replace a none pte with a marker pte if
                 * it's file-backed, so we don't lose the tracking info.
                 */
-               pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval);
+               install_uffd_wp_ptes_if_needed(vma, address, pvmw.pte, nr_pages, pteval);
 
                /* Update high watermark before we lower rss */
                update_hiwater_rss(mm);
--

2.43.0

Does somebody have time to look into that? We should also adjust the doc of pte_install_uffd_wp_if_needed() and turn it into some proper kerneldoc.

Quoted 136 lines · David Hildenbrand (Arm)

Quoted 16 lines · Baolin Wang

As Dev reported[1], it's not ready to support batched unmapping for uffd case. Let's still fallback to per-page unmapping for the uffd case.

[1] https://lore.kernel.org/linux-mm/[email protected]/

Reported-by Dev Jain <[email protected]>

Suggested-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
   mm/rmap.c | 3 +++
   1 file changed, 3 insertions(+)

mm/rmap.c +0 -0

index f13480cb9f2e..172643092dcf 100644

mm/rmap.c static inline unsigned int folio_unmap_pte_batch(struct folio *folio, · line 1958

Quoted 6 lines

mm/rmap.c +3 -0

@@ -1953,6 +1953,9 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
       if (pte_unused(pte))
               return 1;
 
+     if (userfaultfd_wp(vma))
+             return 1;
+

Interesting. I was just wondering why we didn't run into that with lazyfree folios.

Staring at pte_install_uffd_wp_if_needed(), we never set the marker for anonymous VMAs.

So, yeah, if one sets lazyfree on a uffd-wp PTE, the uffd-wp bit will just get zapped alongside. Just like MADV_DONTNEED.

I'm fine with that temporary fix. But I guess the non-hacky way to handle this would be:

From 53d016d6e6f624425dbdbc2fb1dec7c91fbef15c Mon Sep 17 00:00:00 2001
From: "David Hildenbrand (Arm)" <[email protected]>
Date: Mon, 9 Feb 2026 10:52:59 +0100
Subject: [PATCH] tmp

Signed-off-by David Hildenbrand (Arm) <[email protected]>

---
   include/linux/mm_inline.h | 15 ++++++---------
   mm/memory.c               | 21 +--------------------
   mm/rmap.c                 |  2 +-
   3 files changed, 8 insertions(+), 30 deletions(-)

include/linux/mm_inline.h +6 -9

index fa2d6ba811b5..8a9a2c5f5ee3 100644
@@ -566,9 +566,8 @@ static inline pte_marker copy_pte_marker(
    *
    * Returns true if an uffd-wp pte was installed, false otherwise.
    */
-static inline bool
-pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
-                             pte_t *pte, pte_t pteval)
+static inline bool install_uffd_wp_ptes_if_needed(struct vm_area_struct *vma,
+               unsigned long addr, pte_t *pte, unsigned int nr, pte_t pteval)
   {
         bool arm_uffd_pte = false;
 

@@ -598,13 +597,11 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
         if (unlikely(pte_swp_uffd_wp_any(pteval)))
                 arm_uffd_pte = true;
 
-       if (unlikely(arm_uffd_pte)) {
-               set_pte_at(vma->vm_mm, addr, pte,
-                          make_pte_marker(PTE_MARKER_UFFD_WP));
-               return true;
-       }
+       if (likely(!arm_uffd_pte))
+               return false;
 
-       return false;
+       set_ptes(vma->vm_mm, addr, pte, make_pte_marker(PTE_MARKER_UFFD_WP), nr);
+       return true;
   }
 
   static inline bool vma_has_recency(const struct vm_area_struct *vma)

mm/memory.c +1 -20

index da360a6eb8a4..0a87d02a9a69 100644
@@ -1592,29 +1592,10 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
                               unsigned long addr, pte_t *pte, int nr,
                               struct zap_details *details, pte_t pteval)
   {
-       bool was_installed = false;
-
-       if (!uffd_supports_wp_marker())
-               return false;
-
-       /* Zap on anonymous always means dropping everything */
-       if (vma_is_anonymous(vma))
-               return false;
-
         if (zap_drop_markers(details))
                 return false;
 
-       for (;;) {
-               /* the PFN in the PTE is irrelevant. */
-               if (pte_install_uffd_wp_if_needed(vma, addr, pte, pteval))
-                       was_installed = true;
-               if (--nr == 0)
-                       break;
-               pte++;
-               addr += PAGE_SIZE;
-       }
-
-       return was_installed;
+       return install_uffd_wp_ptes_if_needed(vma, addr, pte, nr, pteval);
   }
 
   static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,

mm/rmap.c +1 -2

index 7b9879ef442d..f71aacf35925 100644
@@ -2061,7 +2061,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                  * we may want to replace a none pte with a marker pte if
                  * it's file-backed, so we don't lose the tracking info.
                  */
-               pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval);
+               install_uffd_wp_ptes_if_needed(vma, address, pvmw.pte, nr_pages, pteval);
 
                 /* Update high watermark before we lower rss */
                 update_hiwater_rss(mm);
--

2.43.0

Quoted 2 lines

Does somebody have time to look into that? We should also adjust the doc of pte_install_uffd_wp_if_needed() and turn it into some proper kerneldoc.

I'd nominate Dev, if he has the time, as he has been working on related changes and is already familiar with this area :-)

https://lore.kernel.org/linux-mm/[email protected]/

I assume this could be treated as a separate optimization, as the current temporary fix seems acceptable?

Quoted 134 lines · David Hildenbrand (Arm)

Quoted 16 lines · Baolin Wang

As Dev reported[1], it's not ready to support batched unmapping for uffd case. Let's still fallback to per-page unmapping for the uffd case.

[1] https://lore.kernel.org/linux-mm/[email protected]/

Reported-by Dev Jain <[email protected]>

Suggested-by Barry Song <[email protected]>

Signed-off-by Baolin Wang <[email protected]>

---
  mm/rmap.c | 3 +++
  1 file changed, 3 insertions(+)

mm/rmap.c +0 -0

index f13480cb9f2e..172643092dcf 100644

mm/rmap.c static inline unsigned int folio_unmap_pte_batch(struct folio *folio, · line 1958

Quoted 6 lines

mm/rmap.c +3 -0

@@ -1953,6 +1953,9 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio,
      if (pte_unused(pte))
              return 1;
 
+     if (userfaultfd_wp(vma))
+             return 1;
+

Interesting. I was just wondering why we didn't run into that with lazyfree folios.

Staring at pte_install_uffd_wp_if_needed(), we never set the marker for anonymous VMAs.

So, yeah, if one sets lazyfree on a uffd-wp PTE, the uffd-wp bit will just get zapped alongside. Just like MADV_DONTNEED.

I'm fine with that temporary fix. But I guess the non-hacky way to handle this would be:

From 53d016d6e6f624425dbdbc2fb1dec7c91fbef15c Mon Sep 17 00:00:00 2001
From: "David Hildenbrand (Arm)" <[email protected]>
Date: Mon, 9 Feb 2026 10:52:59 +0100
Subject: [PATCH] tmp

Signed-off-by David Hildenbrand (Arm) <[email protected]>

---
  include/linux/mm_inline.h | 15 ++++++---------
  mm/memory.c               | 21 +--------------------
  mm/rmap.c                 |  2 +-
  3 files changed, 8 insertions(+), 30 deletions(-)

include/linux/mm_inline.h +6 -9

index fa2d6ba811b5..8a9a2c5f5ee3 100644
@@ -566,9 +566,8 @@ static inline pte_marker copy_pte_marker(
   *
   * Returns true if an uffd-wp pte was installed, false otherwise.
   */
-static inline bool
-pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
-                             pte_t *pte, pte_t pteval)
+static inline bool install_uffd_wp_ptes_if_needed(struct vm_area_struct *vma,
+               unsigned long addr, pte_t *pte, unsigned int nr, pte_t pteval)
  {
        bool arm_uffd_pte = false;
 

@@ -598,13 +597,11 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
        if (unlikely(pte_swp_uffd_wp_any(pteval)))
                arm_uffd_pte = true;
 
-       if (unlikely(arm_uffd_pte)) {
-               set_pte_at(vma->vm_mm, addr, pte,
-                          make_pte_marker(PTE_MARKER_UFFD_WP));
-               return true;
-       }
+       if (likely(!arm_uffd_pte))
+               return false;
 
-       return false;
+       set_ptes(vma->vm_mm, addr, pte, make_pte_marker(PTE_MARKER_UFFD_WP), nr);
+       return true;
  }
 
  static inline bool vma_has_recency(const struct vm_area_struct *vma)

mm/memory.c +1 -20

index da360a6eb8a4..0a87d02a9a69 100644
@@ -1592,29 +1592,10 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
                              unsigned long addr, pte_t *pte, int nr,
                              struct zap_details *details, pte_t pteval)
  {
-       bool was_installed = false;
-
-       if (!uffd_supports_wp_marker())
-               return false;
-
-       /* Zap on anonymous always means dropping everything */
-       if (vma_is_anonymous(vma))
-               return false;
-
        if (zap_drop_markers(details))
                return false;
 
-       for (;;) {
-               /* the PFN in the PTE is irrelevant. */
-               if (pte_install_uffd_wp_if_needed(vma, addr, pte, pteval))
-                       was_installed = true;
-               if (--nr == 0)
-                       break;
-               pte++;
-               addr += PAGE_SIZE;
-       }
-
-       return was_installed;
+       return install_uffd_wp_ptes_if_needed(vma, addr, pte, nr, pteval);
  }
 
  static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb,

mm/rmap.c +1 -2

index 7b9879ef442d..f71aacf35925 100644
@@ -2061,7 +2061,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
                 * we may want to replace a none pte with a marker pte if
                 * it's file-backed, so we don't lose the tracking info.
                 */
-               pte_install_uffd_wp_if_needed(vma, address, pvmw.pte, pteval);
+               install_uffd_wp_ptes_if_needed(vma, address, pvmw.pte, nr_pages, pteval);
 
                /* Update high watermark before we lower rss */
                update_hiwater_rss(mm);
--

2.43.0