Skip to contentSkip to conversation

Optimize mremap() for large folios

PATCH Applied Dev Jain 6 participants 25 messages Jun 10, 2025 – Oct 28, 2025

Subsystems (2)

v4 of 5 Applied 2/2 Latest is v5 →

lore
Share & tools
lore

Dev Jain

author
#0 lore raw

v4 cover Optimize mremap() for large folios

Currently move_ptes() iterates through ptes one by one. If the underlying folio mapped by the ptes is large, we can process those ptes in a batch using folio_pte_batch(), thus clearing and setting the PTEs in one go. For arm64 specifically, this results in a 16x reduction in the number of ptep_get() calls (since on a contig block, ptep_get() on arm64 will iterate through all 16 entries to collect a/d bits), and we also elide extra TLBIs through get_and_clear_full_ptes, replacing ptep_get_and_clear.

Mapping 1M of memory with 64K folios, memsetting it, remapping it to src + 1M, and munmapping it 10,000 times, the average execution time reduces from 1.9 to 1.2 seconds, giving a 37% performance optimization, on Apple M3 (arm64). No regression is observed for small folios.

The patchset is based on mm-unstable (6ebffe676fcf).

Test program for reference:

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include <errno.h>

#define SIZE (1UL << 20) // 1M

int main(void) {
    void *new_addr, *addr;

    for (int i = 0; i < 10000; ++i) {
        addr = mmap((void *)(1UL << 30), SIZE, PROT_READ | PROT_WRITE,
                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
        if (addr == MAP_FAILED) {
                perror("mmap");
                return 1;
        }
        memset(addr, 0xAA, SIZE);

        new_addr = mremap(addr, SIZE, SIZE, MREMAP_MAYMOVE | MREMAP_FIXED, addr + SIZE);
        if (new_addr != (addr + SIZE)) {
                perror("mremap");
                return 1;
        }
        munmap(new_addr, SIZE);
    }

}

v3->v4:
- Remove comment above mremap_folio_pte_batch, improve patch description differentiating between folio splitting and pagetable splitting
v2->v3:
- Refactor mremap_folio_pte_batch, drop maybe_contiguous_pte_pfns, fix indentation (Lorenzo), fix cover letter description (512K -> 1M)

v1->v2:
- Expand patch descriptions, move pte declarations to a new line, reduce indentation in patch 2 by introducing mremap_folio_pte_batch(), fix loop iteration (Lorenzo)
- Merge patch 2 and 3 (Anshuman, Lorenzo)
- Fix maybe_contiguous_pte_pfns (Willy)

Dev Jain (2):
mm: Call pointers to ptes as ptep
mm: Optimize mremap() by PTE batching

 mm/mremap.c | 58 ++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 16 deletions(-)
Signature · 1 line
2.30.2
author
#1 lore raw

PATCH v4 1/2 mm: Call pointers to ptes as ptep

Avoid confusion between pte_t* and pte_t data types by suffixing pointer type variables with p. No functional change.

---
 mm/mremap.c | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)
mm/mremap.c +16 -15
index 60f6b8d0d5f0..180b12225368 100644
@@ -176,7 +176,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	struct vm_area_struct *vma = pmc->old;
 	bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
 	struct mm_struct *mm = vma->vm_mm;
-	pte_t *old_pte, *new_pte, pte;
+	pte_t *old_ptep, *new_ptep;
+	pte_t pte;
 	pmd_t dummy_pmdval;
 	spinlock_t *old_ptl, *new_ptl;
 	bool force_flush = false;
@@ -211,8 +212,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	 * We don't have to worry about the ordering of src and dst
 	 * pte locks because exclusive mmap_lock prevents deadlock.
 	 */
-	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
-	if (!old_pte) {
+	old_ptep = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
+	if (!old_ptep) {
 		err = -EAGAIN;
 		goto out;
 	}
@@ -223,10 +224,10 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	 * mmap_lock, so this new_pte page is stable, so there is no need to get
 	 * pmdval and do pmd_same() check.
 	 */
-	new_pte = pte_offset_map_rw_nolock(mm, new_pmd, new_addr, &dummy_pmdval,
+	new_ptep = pte_offset_map_rw_nolock(mm, new_pmd, new_addr, &dummy_pmdval,
 					   &new_ptl);
-	if (!new_pte) {
-		pte_unmap_unlock(old_pte, old_ptl);
+	if (!new_ptep) {
+		pte_unmap_unlock(old_ptep, old_ptl);
 		err = -EAGAIN;
 		goto out;
 	}
@@ -235,14 +236,14 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	flush_tlb_batched_pending(vma->vm_mm);
 	arch_enter_lazy_mmu_mode();
 
-	for (; old_addr < old_end; old_pte++, old_addr += PAGE_SIZE,
-				   new_pte++, new_addr += PAGE_SIZE) {
-		VM_WARN_ON_ONCE(!pte_none(*new_pte));
+	for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
+				   new_ptep++, new_addr += PAGE_SIZE) {
+		VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-		if (pte_none(ptep_get(old_pte)))
+		if (pte_none(ptep_get(old_ptep)))
 			continue;
 
-		pte = ptep_get_and_clear(mm, old_addr, old_pte);
+		pte = ptep_get_and_clear(mm, old_addr, old_ptep);
 		/*
 		 * If we are remapping a valid PTE, make sure
 		 * to flush TLB before we drop the PTL for the
@@ -260,7 +261,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 		pte = move_soft_dirty_pte(pte);
 
 		if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-			pte_clear(mm, new_addr, new_pte);
+			pte_clear(mm, new_addr, new_ptep);
 		else {
 			if (need_clear_uffd_wp) {
 				if (pte_present(pte))
@@ -268,7 +269,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 				else if (is_swap_pte(pte))
 					pte = pte_swp_clear_uffd_wp(pte);
 			}
-			set_pte_at(mm, new_addr, new_pte, pte);
+			set_pte_at(mm, new_addr, new_ptep, pte);
 		}
 	}
 
@@ -277,8 +278,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
 		flush_tlb_range(vma, old_end - len, old_end);
 	if (new_ptl != old_ptl)
 		spin_unlock(new_ptl);
-	pte_unmap(new_pte - 1);
-	pte_unmap_unlock(old_pte - 1, old_ptl);
+	pte_unmap(new_ptep - 1);
+	pte_unmap_unlock(old_ptep - 1, old_ptl);
 out:
 	if (pmc->need_rmap_locks)
 		drop_rmap_locks(vma);
Signature · 1 line
2.30.2
Acked-by on v4 1/2
#2 lore raw

Re: [PATCH v4 1/2] mm: Call pointers to ptes as ptep

Quoted 16 lines · Dev Jain

Avoid confusion between pte_t* and pte_t data types by suffixing pointer type variables with p. No functional change.

---
  mm/mremap.c | 31 ++++++++++++++++---------------
  1 file changed, 16 insertions(+), 15 deletions(-)
mm/mremap.c +0 -0
index 60f6b8d0d5f0..180b12225368 100644

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 180

Quoted 6 lines
mm/mremap.c +2 -1
@@ -176,7 +176,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	struct vm_area_struct *vma = pmc->old;
  	bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
  	struct mm_struct *mm = vma->vm_mm;
-	pte_t *old_pte, *new_pte, pte;
+	pte_t *old_ptep, *new_ptep;
+	pte_t pte;

Could have left that on the same line ...

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 217

Quoted 12 lines
mm/mremap.c +2 -2
  	pmd_t dummy_pmdval;
  	spinlock_t *old_ptl, *new_ptl;
  	bool force_flush = false;
@@ -211,8 +212,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	 * We don't have to worry about the ordering of src and dst
  	 * pte locks because exclusive mmap_lock prevents deadlock.
  	 */
-	old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
-	if (!old_pte) {
+	old_ptep = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
+	if (!old_ptep) {
  		err = -EAGAIN;

One of those things that's completely inconsistent all over the place.

But yeah, I agree that ptep is much better for a PTE pointer.

Signature · 3 lines
Cheers,

David / dhildenb
author
#3 lore raw

Re: [PATCH v4 1/2] mm: Call pointers to ptes as ptep

Quoted 12 lines · David Hildenbrand
Quoted 11 lines · Dev Jain

Avoid confusion between pte_t* and pte_t data types by suffixing pointer type variables with p. No functional change.

---
  mm/mremap.c | 31 ++++++++++++++++---------------

  1 file changed, 16 insertions(+), 15 deletions(-)
Quoted 14 lines

mm/mremap.c static int move_ptes(struct · line 181

Quoted 12 lines
mm/mremap.c +2 -1
index 60f6b8d0d5f0..180b12225368 100644
@@ -176,7 +176,8 @@ static int move_ptes(struct 
 pagetable_move_control *pmc,
       struct vm_area_struct *vma = pmc->old;
       bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
       struct mm_struct *mm = vma->vm_mm;
-    pte_t *old_pte, *new_pte, pte;
+    pte_t *old_ptep, *new_ptep;
+    pte_t pte;

Could have left that on the same line ...

AFAIR Lorenzo had insisted on moving that to a new line.

Quoted 21 lines
Quoted 5 lines
mm/mremap.c +0 -0
       pmd_t dummy_pmdval;
       spinlock_t *old_ptl, *new_ptl;

      bool force_flush = false;

mm/mremap.c static int move_ptes(struct · line 218

Quoted 9 lines
mm/mremap.c +2 -2
@@ -211,8 +212,8 @@ static int move_ptes(struct 
 pagetable_move_control *pmc,
        * We don't have to worry about the ordering of src and dst
        * pte locks because exclusive mmap_lock prevents deadlock.
        */
-    old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
-    if (!old_pte) {
+    old_ptep = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
+    if (!old_ptep) {
           err = -EAGAIN;

One of those things that's completely inconsistent all over the place.

But yeah, I agree that ptep is much better for a PTE pointer.

#4 lore raw

Re: [PATCH v4 1/2] mm: Call pointers to ptes as ptep

Quoted 29 lines · Dev Jain
Quoted 12 lines · David Hildenbrand
Quoted 11 lines · Dev Jain

Avoid confusion between pte_t* and pte_t data types by suffixing pointer type variables with p. No functional change.

---
  mm/mremap.c | 31 ++++++++++++++++---------------

  1 file changed, 16 insertions(+), 15 deletions(-)
Quoted 14 lines

mm/mremap.c static int move_ptes(struct · line 181

Quoted 12 lines
mm/mremap.c +2 -1
index 60f6b8d0d5f0..180b12225368 100644
@@ -176,7 +176,8 @@ static int move_ptes(struct
 pagetable_move_control *pmc,
       struct vm_area_struct *vma = pmc->old;
       bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
       struct mm_struct *mm = vma->vm_mm;
-    pte_t *old_pte, *new_pte, pte;
+    pte_t *old_ptep, *new_ptep;
+    pte_t pte;

Could have left that on the same line ...

Quoted 1 line

AFAIR Lorenzo had insisted on moving that to a new line.

Yeah, not a fan of having pointer and non-pointer types declared on same line.

Quoted 23 lines
Quoted 23 lines
Quoted 7 lines
mm/mremap.c +0 -0
       pmd_t dummy_pmdval;
       spinlock_t *old_ptl, *new_ptl;

      bool force_flush = false;

mm/mremap.c static int move_ptes(struct · line 218

Quoted 9 lines
mm/mremap.c +2 -2
@@ -211,8 +212,8 @@ static int move_ptes(struct
 pagetable_move_control *pmc,
        * We don't have to worry about the ordering of src and dst
        * pte locks because exclusive mmap_lock prevents deadlock.
        */
-    old_pte = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
-    if (!old_pte) {
+    old_ptep = pte_offset_map_lock(mm, old_pmd, old_addr, &old_ptl);
+    if (!old_ptep) {
           err = -EAGAIN;

One of those things that's completely inconsistent all over the place.

But yeah, I agree that ptep is much better for a PTE pointer.

#5 lore raw

Re: [PATCH v4 1/2] mm: Call pointers to ptes as ptep

Quoted 30 lines · Lorenzo Stoakes
Quoted 29 lines · Dev Jain
Quoted 26 lines · David Hildenbrand
Quoted 16 lines · Dev Jain

Avoid confusion between pte_t* and pte_t data types by suffixing pointer type variables with p. No functional change.

---
   mm/mremap.c | 31 ++++++++++++++++---------------
   1 file changed, 16 insertions(+), 15 deletions(-)
mm/mremap.c +0 -0
index 60f6b8d0d5f0..180b12225368 100644

mm/mremap.c static int move_ptes(struct · line 181

Quoted 7 lines
mm/mremap.c +2 -1
@@ -176,7 +176,8 @@ static int move_ptes(struct
 pagetable_move_control *pmc,
       struct vm_area_struct *vma = pmc->old;
       bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
       struct mm_struct *mm = vma->vm_mm;
-    pte_t *old_pte, *new_pte, pte;
+    pte_t *old_ptep, *new_ptep;
+    pte_t pte;

Could have left that on the same line ...

Quoted 3 lines
Quoted 1 line

AFAIR Lorenzo had insisted on moving that to a new line.

Yeah, not a fan of having pointer and non-pointer types declared on same line.

Apparently I am for such simple things.

The joy of not having a common coding style. :)

Anyhow, I don't really care, just something I noticed ...

Signature · 3 lines
Cheers,

David / dhildenb
reviewer Reviewed-by on v4 1/2
#6 lore raw

Re: [PATCH v4 1/2] mm: Call pointers to ptes as ptep

In reply to Dev Jain
Quoted 7 lines · Dev Jain

Avoid confusion between pte_t* and pte_t data types by suffixing pointer type variables with p. No functional change.

Thanks, this is less confusing!

Feel free to add:

Signature · 1 line
Pedro
author
#7 lore raw

PATCH v4 2/2 mm: Optimize mremap() by PTE batching

In reply to Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
 mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +32 -7
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
 	return pte;
 }
 
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t pte, int max_nr)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	struct folio *folio;
+
+	if (max_nr == 1)
+		return 1;
+
+	folio = vm_normal_folio(vma, addr, pte);
+	if (!folio || !folio_test_large(folio))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+			       NULL, NULL);
+}
+
 static int move_ptes(struct pagetable_move_control *pmc,
 		unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
 {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
 	struct mm_struct *mm = vma->vm_mm;
 	pte_t *old_ptep, *new_ptep;
-	pte_t pte;
+	pte_t old_pte, pte;
 	pmd_t dummy_pmdval;
 	spinlock_t *old_ptl, *new_ptl;
 	bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	unsigned long new_addr = pmc->new_addr;
 	unsigned long old_end = old_addr + extent;
 	unsigned long len = old_end - old_addr;
+	int max_nr_ptes;
+	int nr_ptes;
 	int err = 0;
 
 	/*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	flush_tlb_batched_pending(vma->vm_mm);
 	arch_enter_lazy_mmu_mode();
 
-	for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-				   new_ptep++, new_addr += PAGE_SIZE) {
+	for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+		new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
 		VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-		if (pte_none(ptep_get(old_ptep)))
+		nr_ptes = 1;
+		max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+		old_pte = ptep_get(old_ptep);
+		if (pte_none(old_pte))
 			continue;
 
-		pte = ptep_get_and_clear(mm, old_addr, old_ptep);
 		/*
 		 * If we are remapping a valid PTE, make sure
 		 * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
 		 * the TLB entry for the old mapping has been
 		 * flushed.
 		 */
-		if (pte_present(pte))
+		if (pte_present(old_pte)) {
+			nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+							 old_pte, max_nr_ptes);
 			force_flush = true;
+		}
+		pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
 		pte = move_pte(pte, old_addr, new_addr);
 		pte = move_soft_dirty_pte(pte);
 
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 				else if (is_swap_pte(pte))
 					pte = pte_swp_clear_uffd_wp(pte);
 			}
-			set_pte_at(mm, new_addr, new_ptep, pte);
+			set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
 		}
 	}
 
Signature · 1 line
2.30.2
#8 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Hi Dev,

Quoted 30 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
 mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +1 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
        return pte;
 }
 
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 183

Quoted 10 lines
mm/mremap.c +10 -0
+               pte_t *ptep, pte_t pte, int max_nr)
+{
+       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       struct folio *folio;
+
+       if (max_nr == 1)
+               return 1;
+
+       folio = vm_normal_folio(vma, addr, pte);
+       if (!folio || !folio_test_large(folio))

I'm curious about the following case:
If the addr/ptep is not the first subpage of the folio—for example, the 14th subpage—will mremap_folio_pte_batch() return 3?
If so, get_and_clear_full_ptes() would operate on 3 subpages of the folio. In that case, can unfold still work correctly?

Similarly, if the addr/ptep points to the first subpage, but max_nr is less than CONT_PTES, what will happen in that case?

Quoted 72 lines
mm/mremap.c +21 -8
+               return 1;
+
+       return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                              NULL, NULL);
+}
+
 static int move_ptes(struct pagetable_move_control *pmc,
                unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
 {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
        bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
        struct mm_struct *mm = vma->vm_mm;
        pte_t *old_ptep, *new_ptep;
-       pte_t pte;
+       pte_t old_pte, pte;
        pmd_t dummy_pmdval;
        spinlock_t *old_ptl, *new_ptl;
        bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
        unsigned long new_addr = pmc->new_addr;
        unsigned long old_end = old_addr + extent;
        unsigned long len = old_end - old_addr;
+       int max_nr_ptes;
+       int nr_ptes;
        int err = 0;
 
        /*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
        flush_tlb_batched_pending(vma->vm_mm);
        arch_enter_lazy_mmu_mode();
 
-       for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-                                  new_ptep++, new_addr += PAGE_SIZE) {
+       for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+               new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
                VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-               if (pte_none(ptep_get(old_ptep)))
+               nr_ptes = 1;
+               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+               old_pte = ptep_get(old_ptep);
+               if (pte_none(old_pte))
                        continue;
 
-               pte = ptep_get_and_clear(mm, old_addr, old_ptep);
                /*
                 * If we are remapping a valid PTE, make sure
                 * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
                 * the TLB entry for the old mapping has been
                 * flushed.
                 */
-               if (pte_present(pte))
+               if (pte_present(old_pte)) {
+                       nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+                                                        old_pte, max_nr_ptes);
                        force_flush = true;
+               }
+               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
                pte = move_pte(pte, old_addr, new_addr);
                pte = move_soft_dirty_pte(pte);
 
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
                                else if (is_swap_pte(pte))
                                        pte = pte_swp_clear_uffd_wp(pte);
                        }
-                       set_pte_at(mm, new_addr, new_ptep, pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
                }
        }
 
--
Quoted 2 lines

2.30.2

Thanks
Barry

author
#9 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Quoted 31 lines · Barry Song

Hi Dev,

Quoted 28 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +0 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
         return pte;
  }
 
Quoted 14 lines

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 183

Quoted 11 lines
mm/mremap.c +11 -0
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+               pte_t *ptep, pte_t pte, int max_nr)
+{
+       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       struct folio *folio;
+
+       if (max_nr == 1)
+               return 1;
+
+       folio = vm_normal_folio(vma, addr, pte);
+       if (!folio || !folio_test_large(folio))

I'm curious about the following case:
If the addr/ptep is not the first subpage of the folio—for example, the 14th subpage—will mremap_folio_pte_batch() return 3?

It will return the number of PTEs, starting from the PTE pointing to the 14th subpage, that point to consecutive pages of the same large folio, up till max_nr. For an example, if we are operating on a single large folio of order 4, then max_nr will be 16 - 14 + 1 = 3. So in this case we will return 3, since the 14th, 15th and 16th PTE point to consec pages of the same large folio.

Quoted 2 lines

If so, get_and_clear_full_ptes() would operate on 3 subpages of the folio. In that case, can unfold still work correctly?

Yes, first we unfold as in, we do a BBM sequence: cont -> clear -> non-cont. Then, on this non-contig block, we will clear only the PTEs which were asked for us to do.

Quoted 81 lines

Similarly, if the addr/ptep points to the first subpage, but max_nr is less than CONT_PTES, what will happen in that case?

Quoted 72 lines
mm/mremap.c +21 -8
+               return 1;
+
+       return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                              NULL, NULL);
+}
+
  static int move_ptes(struct pagetable_move_control *pmc,
                 unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
  {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
         bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
         struct mm_struct *mm = vma->vm_mm;
         pte_t *old_ptep, *new_ptep;
-       pte_t pte;
+       pte_t old_pte, pte;
         pmd_t dummy_pmdval;
         spinlock_t *old_ptl, *new_ptl;
         bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
         unsigned long new_addr = pmc->new_addr;
         unsigned long old_end = old_addr + extent;
         unsigned long len = old_end - old_addr;
+       int max_nr_ptes;
+       int nr_ptes;
         int err = 0;
 
         /*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
         flush_tlb_batched_pending(vma->vm_mm);
         arch_enter_lazy_mmu_mode();
 
-       for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-                                  new_ptep++, new_addr += PAGE_SIZE) {
+       for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+               new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
                 VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-               if (pte_none(ptep_get(old_ptep)))
+               nr_ptes = 1;
+               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+               old_pte = ptep_get(old_ptep);
+               if (pte_none(old_pte))
                         continue;
 
-               pte = ptep_get_and_clear(mm, old_addr, old_ptep);
                 /*
                  * If we are remapping a valid PTE, make sure
                  * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
                  * the TLB entry for the old mapping has been
                  * flushed.
                  */
-               if (pte_present(pte))
+               if (pte_present(old_pte)) {
+                       nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+                                                        old_pte, max_nr_ptes);
                         force_flush = true;
+               }
+               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
                 pte = move_pte(pte, old_addr, new_addr);
                 pte = move_soft_dirty_pte(pte);
 
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
                                 else if (is_swap_pte(pte))
                                         pte = pte_swp_clear_uffd_wp(pte);
                         }
-                       set_pte_at(mm, new_addr, new_ptep, pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
                 }
         }
 
--
Quoted 2 lines

2.30.2

Thanks
Barry

#10 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Quoted 55 lines · Dev Jain
Quoted 31 lines · Barry Song

Hi Dev,

Quoted 28 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +0 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
         return pte;
  }
 
Quoted 14 lines

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 183

Quoted 11 lines
mm/mremap.c +11 -0
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+               pte_t *ptep, pte_t pte, int max_nr)
+{
+       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       struct folio *folio;
+
+       if (max_nr == 1)
+               return 1;
+
+       folio = vm_normal_folio(vma, addr, pte);
+       if (!folio || !folio_test_large(folio))

I'm curious about the following case:
If the addr/ptep is not the first subpage of the folio—for example, the 14th subpage—will mremap_folio_pte_batch() return 3?

It will return the number of PTEs, starting from the PTE pointing to the 14th subpage, that point to consecutive pages of the same large folio, up till max_nr. For an example, if we are operating on a single large folio of order 4, then max_nr will be 16 - 14 + 1 = 3. So in this case we will return 3, since the 14th, 15th and 16th PTE point to consec pages of the same large folio.

Quoted 6 lines
Quoted 2 lines

If so, get_and_clear_full_ptes() would operate on 3 subpages of the folio. In that case, can unfold still work correctly?

Yes, first we unfold as in, we do a BBM sequence: cont -> clear -> non-cont. Then, on this non-contig block, we will clear only the PTEs which were asked for us to do.

While going through the code,

static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, unsigned int nr, int full)

{
        pte_t pte;
        if (likely(nr == 1)) {
                contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
                pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full);
        } else {
                pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full);
        }

        return pte;
}

Initially, I thought it only unfolded when nr == 1, but after reading contpte_get_and_clear_full_ptes more closely, I realized we do support partial unfolding—that's what I had missed.

pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, unsigned int nr, int full)

{
        contpte_try_unfold_partial(mm, addr, ptep, nr);
        return __get_and_clear_full_ptes(mm, addr, ptep, nr, full);
}

I think you are right.

Quoted 77 lines
Quoted 77 lines

Similarly, if the addr/ptep points to the first subpage, but max_nr is less than CONT_PTES, what will happen in that case?

Quoted 71 lines
mm/mremap.c +21 -7
+               return 1;
+
+       return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                              NULL, NULL);
+}
+
  static int move_ptes(struct pagetable_move_control *pmc,
                 unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
  {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
         bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
         struct mm_struct *mm = vma->vm_mm;
         pte_t *old_ptep, *new_ptep;
-       pte_t pte;
+       pte_t old_pte, pte;
         pmd_t dummy_pmdval;
         spinlock_t *old_ptl, *new_ptl;
         bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
         unsigned long new_addr = pmc->new_addr;
         unsigned long old_end = old_addr + extent;
         unsigned long len = old_end - old_addr;
+       int max_nr_ptes;
+       int nr_ptes;
         int err = 0;
 
         /*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
         flush_tlb_batched_pending(vma->vm_mm);
         arch_enter_lazy_mmu_mode();
 
-       for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-                                  new_ptep++, new_addr += PAGE_SIZE) {
+       for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+               new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
                 VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-               if (pte_none(ptep_get(old_ptep)))
+               nr_ptes = 1;
+               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+               old_pte = ptep_get(old_ptep);
+               if (pte_none(old_pte))
                         continue;
 
-               pte = ptep_get_and_clear(mm, old_addr, old_ptep);
                 /*
                  * If we are remapping a valid PTE, make sure
                  * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
                  * the TLB entry for the old mapping has been
                  * flushed.
                  */
-               if (pte_present(pte))
+               if (pte_present(old_pte)) {
+                       nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+                                                        old_pte, max_nr_ptes);
                         force_flush = true;
+               }
+               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
                 pte = move_pte(pte, old_addr, new_addr);
                 pte = move_soft_dirty_pte(pte);
 
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
                                 else if (is_swap_pte(pte))
                                         pte = pte_swp_clear_uffd_wp(pte);
                         }
-                       set_pte_at(mm, new_addr, new_ptep, pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
                 }
         }
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

--
2.30.2

Thanks
Barry

#11 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Quoted 87 lines · Barry Song
Quoted 55 lines · Dev Jain
Quoted 31 lines · Barry Song

Hi Dev,

Quoted 28 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +0 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
         return pte;
  }
 
Quoted 14 lines

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 183

Quoted 11 lines
mm/mremap.c +11 -0
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+               pte_t *ptep, pte_t pte, int max_nr)
+{
+       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       struct folio *folio;
+
+       if (max_nr == 1)
+               return 1;
+
+       folio = vm_normal_folio(vma, addr, pte);
+       if (!folio || !folio_test_large(folio))

I'm curious about the following case:
If the addr/ptep is not the first subpage of the folio—for example, the 14th subpage—will mremap_folio_pte_batch() return 3?

It will return the number of PTEs, starting from the PTE pointing to the 14th subpage, that point to consecutive pages of the same large folio, up till max_nr. For an example, if we are operating on a single large folio of order 4, then max_nr will be 16 - 14 + 1 = 3. So in this case we will return 3, since the 14th, 15th and 16th PTE point to consec pages of the same large folio.

Quoted 6 lines
Quoted 2 lines

If so, get_and_clear_full_ptes() would operate on 3 subpages of the folio. In that case, can unfold still work correctly?

Yes, first we unfold as in, we do a BBM sequence: cont -> clear -> non-cont. Then, on this non-contig block, we will clear only the PTEs which were asked for us to do.

While going through the code,

static inline pte_t get_and_clear_full_ptes(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, unsigned int nr, int full)

{
        pte_t pte;
        if (likely(nr == 1)) {
                contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep));
                pte = __get_and_clear_full_ptes(mm, addr, ptep, nr, full);
        } else {
                pte = contpte_get_and_clear_full_ptes(mm, addr, ptep, nr, full);
        }

        return pte;
}

Initially, I thought it only unfolded when nr == 1, but after reading contpte_get_and_clear_full_ptes more closely, I realized we do support partial unfolding—that's what I had missed.

pte_t contpte_get_and_clear_full_ptes(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, unsigned int nr, int full)

Quoted 6 lines
{
        contpte_try_unfold_partial(mm, addr, ptep, nr);
        return __get_and_clear_full_ptes(mm, addr, ptep, nr, full);
}

I think you are right.

Yes, Dev is correct; this works as intended. And yes, the code looks a bit dumb, but IIRC, this inline special-casing on nr=1 was needed to prevent fork and/or munmap microbenchmarks from regressing for the common small folio case.

Thanks,
Ryan

Quoted 83 lines
Quoted 78 lines
Quoted 78 lines

Similarly, if the addr/ptep points to the first subpage, but max_nr is less than CONT_PTES, what will happen in that case?

Quoted 71 lines
mm/mremap.c +21 -7
+               return 1;
+
+       return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                              NULL, NULL);
+}
+
  static int move_ptes(struct pagetable_move_control *pmc,
                 unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
  {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
         bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
         struct mm_struct *mm = vma->vm_mm;
         pte_t *old_ptep, *new_ptep;
-       pte_t pte;
+       pte_t old_pte, pte;
         pmd_t dummy_pmdval;
         spinlock_t *old_ptl, *new_ptl;
         bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
         unsigned long new_addr = pmc->new_addr;
         unsigned long old_end = old_addr + extent;
         unsigned long len = old_end - old_addr;
+       int max_nr_ptes;
+       int nr_ptes;
         int err = 0;
 
         /*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
         flush_tlb_batched_pending(vma->vm_mm);
         arch_enter_lazy_mmu_mode();
 
-       for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-                                  new_ptep++, new_addr += PAGE_SIZE) {
+       for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+               new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
                 VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-               if (pte_none(ptep_get(old_ptep)))
+               nr_ptes = 1;
+               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+               old_pte = ptep_get(old_ptep);
+               if (pte_none(old_pte))
                         continue;
 
-               pte = ptep_get_and_clear(mm, old_addr, old_ptep);
                 /*
                  * If we are remapping a valid PTE, make sure
                  * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
                  * the TLB entry for the old mapping has been
                  * flushed.
                  */
-               if (pte_present(pte))
+               if (pte_present(old_pte)) {
+                       nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+                                                        old_pte, max_nr_ptes);
                         force_flush = true;
+               }
+               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
                 pte = move_pte(pte, old_addr, new_addr);
                 pte = move_soft_dirty_pte(pte);
 
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
                                 else if (is_swap_pte(pte))
                                         pte = pte_swp_clear_uffd_wp(pte);
                         }
-                       set_pte_at(mm, new_addr, new_ptep, pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
                 }
         }
 
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

--
2.30.2

Thanks
Barry

Reviewed-by on v4 2/2
#12 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

In reply to Dev Jain
Quoted 10 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

Quoted 7 lines

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

The code appears correct to me:

Quoted 95 lines
---
 mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +32 -8
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
        return pte;
 }
 
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+               pte_t *ptep, pte_t pte, int max_nr)
+{
+       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       struct folio *folio;
+
+       if (max_nr == 1)
+               return 1;
+
+       folio = vm_normal_folio(vma, addr, pte);
+       if (!folio || !folio_test_large(folio))
+               return 1;
+
+       return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                              NULL, NULL);
+}
+
 static int move_ptes(struct pagetable_move_control *pmc,
                unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
 {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
        bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
        struct mm_struct *mm = vma->vm_mm;
        pte_t *old_ptep, *new_ptep;
-       pte_t pte;
+       pte_t old_pte, pte;
        pmd_t dummy_pmdval;
        spinlock_t *old_ptl, *new_ptl;
        bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
        unsigned long new_addr = pmc->new_addr;
        unsigned long old_end = old_addr + extent;
        unsigned long len = old_end - old_addr;
+       int max_nr_ptes;
+       int nr_ptes;
        int err = 0;
 
        /*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
        flush_tlb_batched_pending(vma->vm_mm);
        arch_enter_lazy_mmu_mode();
 
-       for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-                                  new_ptep++, new_addr += PAGE_SIZE) {
+       for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+               new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
                VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-               if (pte_none(ptep_get(old_ptep)))
+               nr_ptes = 1;
+               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+               old_pte = ptep_get(old_ptep);
+               if (pte_none(old_pte))
                        continue;
 
-               pte = ptep_get_and_clear(mm, old_addr, old_ptep);
                /*
                 * If we are remapping a valid PTE, make sure
                 * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
                 * the TLB entry for the old mapping has been
                 * flushed.
                 */
-               if (pte_present(pte))
+               if (pte_present(old_pte)) {
+                       nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+                                                        old_pte, max_nr_ptes);
                        force_flush = true;
+               }
+               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
                pte = move_pte(pte, old_addr, new_addr);
                pte = move_soft_dirty_pte(pte);
 
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
                                else if (is_swap_pte(pte))
                                        pte = pte_swp_clear_uffd_wp(pte);
                        }
-                       set_pte_at(mm, new_addr, new_ptep, pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
                }
        }
 
--
Quoted 2 lines

2.30.2

Thanks

Barry

Reviewed-by on v4 2/2
#13 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

In reply to Dev Jain
Quoted 9 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

Quoted 5 lines

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

This is a fantastic improvement on commit message, thank you very much!

Quoted 2 lines

Looks good to me:

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 172

Quoted 12 lines
---
 mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +0 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
 	return pte;
 }
 

Just for ease of reviewing the review :P

We dropped the comment here as agreed:

Quoted 39 lines
mm/mremap.c +20 -1
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t pte, int max_nr)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	struct folio *folio;
+
+	if (max_nr == 1)
+		return 1;
+
+	folio = vm_normal_folio(vma, addr, pte);
+	if (!folio || !folio_test_large(folio))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+			       NULL, NULL);
+}
+
 static int move_ptes(struct pagetable_move_control *pmc,
 		unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
 {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
 	struct mm_struct *mm = vma->vm_mm;
 	pte_t *old_ptep, *new_ptep;
-	pte_t pte;
+	pte_t old_pte, pte;
 	pmd_t dummy_pmdval;
 	spinlock_t *old_ptl, *new_ptl;
 	bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	unsigned long new_addr = pmc->new_addr;
 	unsigned long old_end = old_addr + extent;
 	unsigned long len = old_end - old_addr;
+	int max_nr_ptes;
+	int nr_ptes;
 	int err = 0;
 
 	/*

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 260

Quoted 8 lines
mm/mremap.c +2 -2
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
 	flush_tlb_batched_pending(vma->vm_mm);
 	arch_enter_lazy_mmu_mode();
 
-	for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-				   new_ptep++, new_addr += PAGE_SIZE) {
+	for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+		new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
 		VM_WARN_ON_ONCE(!pte_none(*new_ptep));

We added this, which does in fact seem sane. We absolutely should not expect to see a new PTE here.

The uprobe logic recently addressed actually made it possible for something to be there (ugh god) but that's now fixed.

Quoted 38 lines
mm/mremap.c +10 -5
-		if (pte_none(ptep_get(old_ptep)))
+		nr_ptes = 1;
+		max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+		old_pte = ptep_get(old_ptep);
+		if (pte_none(old_pte))
 			continue;
 
-		pte = ptep_get_and_clear(mm, old_addr, old_ptep);
 		/*
 		 * If we are remapping a valid PTE, make sure
 		 * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
 		 * the TLB entry for the old mapping has been
 		 * flushed.
 		 */
-		if (pte_present(pte))
+		if (pte_present(old_pte)) {
+			nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+							 old_pte, max_nr_ptes);
 			force_flush = true;
+		}
+		pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
 		pte = move_pte(pte, old_addr, new_addr);
 		pte = move_soft_dirty_pte(pte);
 
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
 				else if (is_swap_pte(pte))
 					pte = pte_swp_clear_uffd_wp(pte);
 			}
-			set_pte_at(mm, new_addr, new_ptep, pte);
+			set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
 		}
 	}
 
--

2.30.2

#14 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

In reply to Dev Jain
Quoted 102 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +31 -6
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
  	return pte;
  }
 
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t pte, int max_nr)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	struct folio *folio;
+
+	if (max_nr == 1)
+		return 1;
+
+	folio = vm_normal_folio(vma, addr, pte);
+	if (!folio || !folio_test_large(folio))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+			       NULL, NULL);
+}
+
  static int move_ptes(struct pagetable_move_control *pmc,
  		unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
  {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
  	struct mm_struct *mm = vma->vm_mm;
  	pte_t *old_ptep, *new_ptep;
-	pte_t pte;
+	pte_t old_pte, pte;
  	pmd_t dummy_pmdval;
  	spinlock_t *old_ptl, *new_ptl;
  	bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	unsigned long new_addr = pmc->new_addr;
  	unsigned long old_end = old_addr + extent;
  	unsigned long len = old_end - old_addr;
+	int max_nr_ptes;
+	int nr_ptes;
  	int err = 0;
 
  	/*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	flush_tlb_batched_pending(vma->vm_mm);
  	arch_enter_lazy_mmu_mode();
 
-	for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-				   new_ptep++, new_addr += PAGE_SIZE) {
+	for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+		new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
  		VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
-		if (pte_none(ptep_get(old_ptep)))
+		nr_ptes = 1;
+		max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+		old_pte = ptep_get(old_ptep);
+		if (pte_none(old_pte))
  			continue;
 
-		pte = ptep_get_and_clear(mm, old_addr, old_ptep);
  		/*
  		 * If we are remapping a valid PTE, make sure
  		 * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
  		 * the TLB entry for the old mapping has been
  		 * flushed.
  		 */
-		if (pte_present(pte))
+		if (pte_present(old_pte)) {
+			nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+							 old_pte, max_nr_ptes);
  			force_flush = true;
+		}
+		pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
  		pte = move_pte(pte, old_addr, new_addr);
  		pte = move_soft_dirty_pte(pte);
 

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 297

Quoted 5 lines
mm/mremap.c +1 -1
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
  				else if (is_swap_pte(pte))
  					pte = pte_swp_clear_uffd_wp(pte);
  			}
-			set_pte_at(mm, new_addr, new_ptep, pte);
+			set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);

What I dislike is that some paths work on a single PTE, and we implicitly have to know that they don't apply for !pte_present.

Like

	if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))

Will not get batched yet. And that is hidden inside the pte_marker_uffd_wp check ...

Should we properly separate both paths (present vs. !present), and while at it, do some more cleanups? I'm thinking of the following on top (only compile-tested)

mm/mremap.c +23 -29
index 18b215521adae..b88abf02b34e0 100644
@@ -155,21 +155,6 @@ static void drop_rmap_locks(struct vm_area_struct *vma)
                i_mmap_unlock_write(vma->vm_file->f_mapping);
 }
 
-static pte_t move_soft_dirty_pte(pte_t pte)
-{
-       /*
-        * Set soft dirty bit so we can notice
-        * in userspace the ptes were moved.
-        */
-#ifdef CONFIG_MEM_SOFT_DIRTY
-       if (pte_present(pte))
-               pte = pte_mksoft_dirty(pte);
-       else if (is_swap_pte(pte))
-               pte = pte_swp_mksoft_dirty(pte);
-#endif
-       return pte;
-}
-
 static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
                pte_t *ptep, pte_t pte, int max_nr)
 {
@@ -260,7 +245,6 @@ static int move_ptes(struct pagetable_move_control *pmc,
                VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
                nr_ptes = 1;
-               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                old_pte = ptep_get(old_ptep);
                if (pte_none(old_pte))
                        continue;
@@ -277,24 +261,34 @@ static int move_ptes(struct pagetable_move_control *pmc,
                 * flushed.
                 */
                if (pte_present(old_pte)) {
+                       max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                        nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
                                                         old_pte, max_nr_ptes);
                        force_flush = true;
-               }
-               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
-               pte = move_pte(pte, old_addr, new_addr);
-               pte = move_soft_dirty_pte(pte);
-
-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-                       pte_clear(mm, new_addr, new_ptep);
-               else {
-                       if (need_clear_uffd_wp) {
-                               if (pte_present(pte))
-                                       pte = pte_clear_uffd_wp(pte);
-                               else if (is_swap_pte(pte))
+
+                       pte = get_and_clear_full_ptes(mm, old_addr, old_ptep,
+                                                     nr_ptes, 0);
+                       /*
+                        * Moving present PTEs requires special care on some
+                        * archs.
+                        */
+                       pte = move_pte(pte, old_addr, new_addr);
+                       /* make userspace aware that this pte moved. */
+                       pte = pte_mksoft_dirty(pte);
+                       if (need_clear_uffd_wp)
+                               pte = pte_clear_uffd_wp(pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
+               } else if (need_clear_uffd_wp && pte_marker_uffd_wp(pte)) {
+                       pte_clear(mm, old_addr, old_ptep);
+               } else {
+                       pte_clear(mm, old_addr, old_ptep);
+                       if (is_swap_pte(pte)) {
+                               if (need_clear_uffd_wp)
                                        pte = pte_swp_clear_uffd_wp(pte);
+                               /* make userspace aware that this pte moved. */
+                               pte = pte_swp_mksoft_dirty(pte);
                        }
-                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
+                       set_pte_at(mm, new_addr, new_ptep, pte);
                }
        }
 

Note that I don't know why we had the existing

- if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
- pte_clear(mm, new_addr, new_ptep);

I thought we would always expect that the destination pte is already pte_none() ?

Signature · 3 lines
Cheers,

David / dhildenb
author
#15 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Quoted 113 lines · David Hildenbrand
Quoted 100 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------

  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +17 -1
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
       return pte;
   }
   +static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+        pte_t *ptep, pte_t pte, int max_nr)
+{
+    const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+    struct folio *folio;
+
+    if (max_nr == 1)
+        return 1;
+
+    folio = vm_normal_folio(vma, addr, pte);
+    if (!folio || !folio_test_large(folio))
+        return 1;
+
+    return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                   NULL, NULL);
+}
+
   static int move_ptes(struct pagetable_move_control *pmc,
           unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
   {
@@ -177,7 +194,7 @@ static int move_ptes(struct 
 pagetable_move_control *pmc,
       bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
       struct mm_struct *mm = vma->vm_mm;
       pte_t *old_ptep, *new_ptep;
-    pte_t pte;
+    pte_t old_pte, pte;
       pmd_t dummy_pmdval;
       spinlock_t *old_ptl, *new_ptl;

      bool force_flush = false;

mm/mremap.c +8 -1
@@ -185,6 +202,8 @@ static int move_ptes(struct 
 pagetable_move_control *pmc,
       unsigned long new_addr = pmc->new_addr;
       unsigned long old_end = old_addr + extent;
       unsigned long len = old_end - old_addr;
+    int max_nr_ptes;
+    int nr_ptes;
       int err = 0;
         /*
@@ -236,14 +255,16 @@ static int move_ptes(struct 
 pagetable_move_control *pmc,
       flush_tlb_batched_pending(vma->vm_mm);
       arch_enter_lazy_mmu_mode();
   -    for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-                   new_ptep++, new_addr += PAGE_SIZE) {
+    for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += 
 nr_ptes * PAGE_SIZE,
+        new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
           VM_WARN_ON_ONCE(!pte_none(*new_ptep));
   -        if (pte_none(ptep_get(old_ptep)))
+        nr_ptes = 1;
+        max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+        old_pte = ptep_get(old_ptep);
+        if (pte_none(old_pte))
               continue;
   -        pte = ptep_get_and_clear(mm, old_addr, old_ptep);
           /*

           * If we are remapping a valid PTE, make sure
           * to flush TLB before we drop the PTL for the

mm/mremap.c +5 -1
@@ -255,8 +276,12 @@ static int move_ptes(struct 
 pagetable_move_control *pmc,
            * the TLB entry for the old mapping has been
            * flushed.
            */
-        if (pte_present(pte))
+        if (pte_present(old_pte)) {
+            nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+                             old_pte, max_nr_ptes);
               force_flush = true;
+        }
+        pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, 
 nr_ptes, 0);
           pte = move_pte(pte, old_addr, new_addr);
Quoted 7 lines

          pte = move_soft_dirty_pte(pte);
  @@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc, else if (is_swap_pte(pte))
                      pte = pte_swp_clear_uffd_wp(pte);
              }
-            set_pte_at(mm, new_addr, new_ptep, pte);
+            set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);

What I dislike is that some paths work on a single PTE, and we implicitly have to know that they don't apply for !pte_present.

Quoted 7 lines

Like
    if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))

Will not get batched yet. And that is hidden inside the pte_marker_uffd_wp check ...

Should we properly separate both paths (present vs. !present), and while at it, do some more cleanups? I'm thinking of the following on top (only compile-tested)

Good observation! Just one doubt, see below.

Quoted 72 lines
mm/mremap.c +0 -15
index 18b215521adae..b88abf02b34e0 100644
@@ -155,21 +155,6 @@ static void drop_rmap_locks(struct vm_area_struct 
 *vma)
 i_mmap_unlock_write(vma->vm_file->f_mapping);
  }
 
-static pte_t move_soft_dirty_pte(pte_t pte)
-{
-       /*
-        * Set soft dirty bit so we can notice
-        * in userspace the ptes were moved.
-        */
-#ifdef CONFIG_MEM_SOFT_DIRTY
-       if (pte_present(pte))
-               pte = pte_mksoft_dirty(pte);
-       else if (is_swap_pte(pte))
-               pte = pte_swp_mksoft_dirty(pte);
-#endif
-       return pte;
-}
-
  static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
                 pte_t *ptep, pte_t pte, int max_nr)

 {

mm/mremap.c +0 -1
@@ -260,7 +245,6 @@ static int move_ptes(struct pagetable_move_control 
 *pmc,
                 VM_WARN_ON_ONCE(!pte_none(*new_ptep));
 
                 nr_ptes = 1;
-               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                 old_pte = ptep_get(old_ptep);
                 if (pte_none(old_pte))

                        continue;

mm/mremap.c +6 -12
@@ -277,24 +261,34 @@ static int move_ptes(struct 
 pagetable_move_control *pmc,
                  * flushed.
                  */
                 if (pte_present(old_pte)) {
+                       max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                         nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
                                                          old_pte, max_nr_ptes);
                         force_flush = true;
-               }
-               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, 
 nr_ptes, 0);
-               pte = move_pte(pte, old_addr, new_addr);
-               pte = move_soft_dirty_pte(pte);
-
-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-                       pte_clear(mm, new_addr, new_ptep);
-               else {
-                       if (need_clear_uffd_wp) {
-                               if (pte_present(pte))
-                                       pte = pte_clear_uffd_wp(pte);
-                               else if (is_swap_pte(pte))
+
+                       pte = get_and_clear_full_ptes(mm, old_addr, 
 old_ptep,
+                                                     nr_ptes, 0);
+                       /*
+                        * Moving present PTEs requires special care 

on some
+                        * archs.
+                        */
+                       pte = move_pte(pte, old_addr, new_addr); +                       /* make userspace aware that this pte moved. */ +                       pte = pte_mksoft_dirty(pte);

Quoted 8 lines

+                       if (need_clear_uffd_wp)
+                               pte = pte_clear_uffd_wp(pte); +                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); +               } else if (need_clear_uffd_wp && pte_marker_uffd_wp(pte)) {
+                       pte_clear(mm, old_addr, old_ptep);
+               } else {
+                       pte_clear(mm, old_addr, old_ptep);

Should pte_clear be included here? It is currently being done only for the case

need_clear_uffd_wp && pte_marker_uffd_wp().

Quoted 22 lines

+ if (is_swap_pte(pte)) {
+                               if (need_clear_uffd_wp)
                                        pte = pte_swp_clear_uffd_wp(pte); +                               /* make userspace aware that this pte moved. */
+                               pte = pte_swp_mksoft_dirty(pte); }
-                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); +                       set_pte_at(mm, new_addr, new_ptep, pte); }
        }

Note that I don't know why we had the existing

-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte)) -                       pte_clear(mm, new_addr, new_ptep);

I thought we would always expect that the destination pte is already pte_none() ?

#16 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Quoted 37 lines
Quoted 25 lines

-               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
-               pte = move_pte(pte, old_addr, new_addr);
-               pte = move_soft_dirty_pte(pte);

mm/mremap.c +2 -8
-
-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-                       pte_clear(mm, new_addr, new_ptep);
-               else {
-                       if (need_clear_uffd_wp) {
-                               if (pte_present(pte))
-                                       pte = pte_clear_uffd_wp(pte);
-                               else if (is_swap_pte(pte))
+
+                       pte = get_and_clear_full_ptes(mm, old_addr,

old_ptep,
+                                                     nr_ptes, 0); +                       /*
+                        * Moving present PTEs requires special care on some
+                        * archs.
+                        */
+                       pte = move_pte(pte, old_addr, new_addr); +                       /* make userspace aware that this pte moved. */ +                       pte = pte_mksoft_dirty(pte);

Quoted 8 lines

+                       if (need_clear_uffd_wp)
+                               pte = pte_clear_uffd_wp(pte); +                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes); +               } else if (need_clear_uffd_wp && pte_marker_uffd_wp(pte)) {
+                       pte_clear(mm, old_addr, old_ptep);
+               } else {
+                       pte_clear(mm, old_addr, old_ptep);

Should pte_clear be included here? It is currently being done only for the case

Quoted 1 line

need_clear_uffd_wp && pte_marker_uffd_wp().

We always cleared the old pte (using get_and_clear_*, which is not required if we already know that it is !present).

The case you mean was what I describe here:

Quoted 11 lines
Quoted 11 lines

Note that I don't know why we had the existing

-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte)) -                       pte_clear(mm, new_addr, new_ptep);

I thought we would always expect that the destination pte is already pte_none() ?

So clearing the destination PTE can be dropped if we didn't move in the first place.

Signature · 3 lines
Cheers,

David / dhildenb
#17 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

In reply to David Hildenbrand
Quoted 98 lines · David Hildenbrand
Quoted 97 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +31 -6
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
  	return pte;
  }
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t pte, int max_nr)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	struct folio *folio;
+
+	if (max_nr == 1)
+		return 1;
+
+	folio = vm_normal_folio(vma, addr, pte);
+	if (!folio || !folio_test_large(folio))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+			       NULL, NULL);
+}
+
  static int move_ptes(struct pagetable_move_control *pmc,
  		unsigned long extent, pmd_t *old_pmd, pmd_t *new_pmd)
  {
@@ -177,7 +194,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	bool need_clear_uffd_wp = vma_has_uffd_without_event_remap(vma);
  	struct mm_struct *mm = vma->vm_mm;
  	pte_t *old_ptep, *new_ptep;
-	pte_t pte;
+	pte_t old_pte, pte;
  	pmd_t dummy_pmdval;
  	spinlock_t *old_ptl, *new_ptl;
  	bool force_flush = false;
@@ -185,6 +202,8 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	unsigned long new_addr = pmc->new_addr;
  	unsigned long old_end = old_addr + extent;
  	unsigned long len = old_end - old_addr;
+	int max_nr_ptes;
+	int nr_ptes;
  	int err = 0;
  	/*
@@ -236,14 +255,16 @@ static int move_ptes(struct pagetable_move_control *pmc,
  	flush_tlb_batched_pending(vma->vm_mm);
  	arch_enter_lazy_mmu_mode();
-	for (; old_addr < old_end; old_ptep++, old_addr += PAGE_SIZE,
-				   new_ptep++, new_addr += PAGE_SIZE) {
+	for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE,
+		new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) {
  		VM_WARN_ON_ONCE(!pte_none(*new_ptep));
-		if (pte_none(ptep_get(old_ptep)))
+		nr_ptes = 1;
+		max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
+		old_pte = ptep_get(old_ptep);
+		if (pte_none(old_pte))
  			continue;
-		pte = ptep_get_and_clear(mm, old_addr, old_ptep);
  		/*
  		 * If we are remapping a valid PTE, make sure
  		 * to flush TLB before we drop the PTL for the
@@ -255,8 +276,12 @@ static int move_ptes(struct pagetable_move_control *pmc,
  		 * the TLB entry for the old mapping has been
  		 * flushed.
  		 */
-		if (pte_present(pte))
+		if (pte_present(old_pte)) {
+			nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
+							 old_pte, max_nr_ptes);
  			force_flush = true;
+		}
+		pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
  		pte = move_pte(pte, old_addr, new_addr);
  		pte = move_soft_dirty_pte(pte);
@@ -269,7 +294,7 @@ static int move_ptes(struct pagetable_move_control *pmc,
  				else if (is_swap_pte(pte))
Quoted 8 lines

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc,

Quoted 4 lines
mm/mremap.c +1 -1
  					pte = pte_swp_clear_uffd_wp(pte);
  			}
-			set_pte_at(mm, new_addr, new_ptep, pte);
+			set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);

What I dislike is that some paths work on a single PTE, and we implicitly have to know that they don't apply for !pte_present.

I hate any kind of implicit knowledge like this.

Quoted 3 lines

Like

	if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))

I also despise [with words I cannot use on-list] how uffd is implemented.

It's _nothing but_ ad-hoc stuff like this spawned all around the place.

It's hateful.

Quoted 5 lines

Will not get batched yet. And that is hidden inside the pte_marker_uffd_wp check ...

Should we properly separate both paths (present vs. !present), and while at it, do some more cleanups? I'm thinking of the following on top (only compile-tested)

I'd like to see that, but I think maybe better as a follow up series?

On the other hand, this does improve this quite a bit. Could also be another patch in the series.

Quoted 52 lines
mm/mremap.c +1 -26
index 18b215521adae..b88abf02b34e0 100644
@@ -155,21 +155,6 @@ static void drop_rmap_locks(struct vm_area_struct *vma)
                i_mmap_unlock_write(vma->vm_file->f_mapping);
 }
-static pte_t move_soft_dirty_pte(pte_t pte)
-{
-       /*
-        * Set soft dirty bit so we can notice
-        * in userspace the ptes were moved.
-        */
-#ifdef CONFIG_MEM_SOFT_DIRTY
-       if (pte_present(pte))
-               pte = pte_mksoft_dirty(pte);
-       else if (is_swap_pte(pte))
-               pte = pte_swp_mksoft_dirty(pte);
-#endif
-       return pte;
-}
-
 static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
                pte_t *ptep, pte_t pte, int max_nr)
 {
@@ -260,7 +245,6 @@ static int move_ptes(struct pagetable_move_control *pmc,
                VM_WARN_ON_ONCE(!pte_none(*new_ptep));
                nr_ptes = 1;
-               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                old_pte = ptep_get(old_ptep);
                if (pte_none(old_pte))
                        continue;
@@ -277,24 +261,34 @@ static int move_ptes(struct pagetable_move_control *pmc,
                 * flushed.
                 */
                if (pte_present(old_pte)) {
+                       max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                        nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
                                                         old_pte, max_nr_ptes);
                        force_flush = true;
-               }
-               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
-               pte = move_pte(pte, old_addr, new_addr);
-               pte = move_soft_dirty_pte(pte);
-
-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-                       pte_clear(mm, new_addr, new_ptep);
-               else {
-                       if (need_clear_uffd_wp) {
-                               if (pte_present(pte))

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 275

Quoted 10 lines
mm/mremap.c +8 -2
-                                       pte = pte_clear_uffd_wp(pte);
-                               else if (is_swap_pte(pte))
+
+                       pte = get_and_clear_full_ptes(mm, old_addr, old_ptep,
+                                                     nr_ptes, 0);
+                       /*
+                        * Moving present PTEs requires special care on some
+                        * archs.
+                        */
+                       pte = move_pte(pte, old_addr, new_addr);

I guess we're good with only doing this in pte_present() case because the only arch that implements this, sparc, does a present check anyway.

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 282

Quoted 7 lines
mm/mremap.c +7 -0
+                       /* make userspace aware that this pte moved. */
+                       pte = pte_mksoft_dirty(pte);
+                       if (need_clear_uffd_wp)
+                               pte = pte_clear_uffd_wp(pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
+               } else if (need_clear_uffd_wp && pte_marker_uffd_wp(pte)) {
+                       pte_clear(mm, old_addr, old_ptep);

Same comment as below re: pte_clear().

I see you've dropped pte_clear(mm, new_addr, new_ptep) which I guess is purposefully?

I do think that it is pointless yes.

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 284

Quoted 2 lines
mm/mremap.c +2 -0
+               } else {
+                       pte_clear(mm, old_addr, old_ptep);

I guess this is intended to replace ptep_get_and_clear_full_ptes() above in the single PTE case... no? Is this sufficient?

In the original code we'd always do ptep_get_and_clear().

I think the key difference is page_table_check_pte_clear().

I notice, hilariously, that there is a ptep_clear() that _does_ call this. What a mess.

Quoted 18 lines
mm/mremap.c +5 -1
+                       if (is_swap_pte(pte)) {
+                               if (need_clear_uffd_wp)
                                        pte = pte_swp_clear_uffd_wp(pte);
+                               /* make userspace aware that this pte moved. */
+                               pte = pte_swp_mksoft_dirty(pte);
                        }
-                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
+                       set_pte_at(mm, new_addr, new_ptep, pte);
                }
        }
 

Note that I don't know why we had the existing

mm/mremap.c +0 -2
-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-                       pte_clear(mm, new_addr, new_ptep);
Quoted 1 line

I thought we would always expect that the destination pte is already pte_none() ?

I think this is because we already did the move_pte() call in the original code before checking this:

		pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
		pte = move_pte(pte, old_addr, new_addr);
		pte = move_soft_dirty_pte(pte);

		if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
			pte_clear(mm, new_addr, new_ptep);

But maybe it's because there was a presumption move_pte() would like you know, move a PTE entry? Which it doesn't, it - only on SPARC - does a hook to flush the dcache.

Quoted 6 lines

--
Cheers,

David / dhildenb

#18 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

[...]

Quoted 21 lines
Quoted 7 lines

Review

Quoted 3 lines
   			}
(quoted diff) +1 -1
-			set_pte_at(mm, new_addr, new_ptep, pte);
+			set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);

What I dislike is that some paths work on a single PTE, and we implicitly have to know that they don't apply for !pte_present.

I hate any kind of implicit knowledge like this.

Quoted 3 lines

Like

	if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))

I also despise [with words I cannot use on-list] how uffd is implemented.

It's _nothing but_ ad-hoc stuff like this spawned all around the place.

It's hateful.

Quoted 1 line
Quoted 6 lines
Quoted 4 lines

Will not get batched yet. And that is hidden inside the pte_marker_uffd_wp check ...

Should we properly separate both paths (present vs. !present), and while at it, do some more cleanups? I'm thinking of the following on top (only compile-tested)

I'd like to see that, but I think maybe better as a follow up series?

I'd do the split as a cleanup patch upfront.

Then add the batching for the pte_present() case.

Quoted 61 lines

On the other hand, this does improve this quite a bit. Could also be another patch in the series.

Quoted 57 lines
mm/mremap.c +4 -28
index 18b215521adae..b88abf02b34e0 100644
@@ -155,21 +155,6 @@ static void drop_rmap_locks(struct vm_area_struct *vma)
                 i_mmap_unlock_write(vma->vm_file->f_mapping);
  }
-static pte_t move_soft_dirty_pte(pte_t pte)
-{
-       /*
-        * Set soft dirty bit so we can notice
-        * in userspace the ptes were moved.
-        */
-#ifdef CONFIG_MEM_SOFT_DIRTY
-       if (pte_present(pte))
-               pte = pte_mksoft_dirty(pte);
-       else if (is_swap_pte(pte))
-               pte = pte_swp_mksoft_dirty(pte);
-#endif
-       return pte;
-}
-
  static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
                 pte_t *ptep, pte_t pte, int max_nr)
  {
@@ -260,7 +245,6 @@ static int move_ptes(struct pagetable_move_control *pmc,
                 VM_WARN_ON_ONCE(!pte_none(*new_ptep));
                 nr_ptes = 1;
-               max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                 old_pte = ptep_get(old_ptep);
                 if (pte_none(old_pte))
                         continue;
@@ -277,24 +261,34 @@ static int move_ptes(struct pagetable_move_control *pmc,
                  * flushed.
                  */
                 if (pte_present(old_pte)) {
+                       max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT;
                         nr_ptes = mremap_folio_pte_batch(vma, old_addr, old_ptep,
                                                          old_pte, max_nr_ptes);
                         force_flush = true;
-               }
-               pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
-               pte = move_pte(pte, old_addr, new_addr);
-               pte = move_soft_dirty_pte(pte);
-
-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-                       pte_clear(mm, new_addr, new_ptep);
-               else {
-                       if (need_clear_uffd_wp) {
-                               if (pte_present(pte))
-                                       pte = pte_clear_uffd_wp(pte);
-                               else if (is_swap_pte(pte))
+
+                       pte = get_and_clear_full_ptes(mm, old_addr, old_ptep,
+                                                     nr_ptes, 0);
Quoted 8 lines

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 275

Quoted 5 lines
mm/mremap.c +5 -0
+                       /*
+                        * Moving present PTEs requires special care on some
+                        * archs.
+                        */
+                       pte = move_pte(pte, old_addr, new_addr);

I guess we're good with only doing this in pte_present() case because the only arch that implements this, sparc, does a present check anyway.

Yes, we could then remove the call from pte_present(), see below.

Quoted 9 lines

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 282

Quoted 8 lines
mm/mremap.c +7 -0
+                       /* make userspace aware that this pte moved. */
+                       pte = pte_mksoft_dirty(pte);
+                       if (need_clear_uffd_wp)
+                               pte = pte_clear_uffd_wp(pte);
+                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
+               } else if (need_clear_uffd_wp && pte_marker_uffd_wp(pte)) {
+                       pte_clear(mm, old_addr, old_ptep);
Quoted 6 lines

Same comment as below re: pte_clear().

I see you've dropped pte_clear(mm, new_addr, new_ptep) which I guess is purposefully?

I do think that it is pointless yes.

Yeah, that's what I raised below.

Quoted 7 lines

mm/mremap.c static int move_ptes(struct pagetable_move_control *pmc, · line 284

Quoted 3 lines
mm/mremap.c +2 -0
+               } else {
+                       pte_clear(mm, old_addr, old_ptep);

I guess this is intended to replace ptep_get_and_clear_full_ptes() above in the single PTE case... no? Is this sufficient?

Quoted 6 lines

In the original code we'd always do ptep_get_and_clear().

I think the key difference is page_table_check_pte_clear().

I notice, hilariously, that there is a ptep_clear() that _does_ call this. What a mess.

ptep_get_and_clear() is only relevant when something is present and could change concurrently (especially A/D bits managed by HW).

We already obtained the pte, it's not present, and now just want to clear it.

Quoted 16 lines
Quoted 16 lines
mm/mremap.c +5 -1
+                       if (is_swap_pte(pte)) {
+                               if (need_clear_uffd_wp)
                                         pte = pte_swp_clear_uffd_wp(pte);
+                               /* make userspace aware that this pte moved. */
+                               pte = pte_swp_mksoft_dirty(pte);
                         }
-                       set_ptes(mm, new_addr, new_ptep, pte, nr_ptes);
+                       set_pte_at(mm, new_addr, new_ptep, pte);
                 }
         }
 

Note that I don't know why we had the existing

Quoted 8 lines
Quoted 5 lines
mm/mremap.c +0 -2
-               if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
-                       pte_clear(mm, new_addr, new_ptep);

I thought we would always expect that the destination pte is already pte_none() ?

I think this is because we already did the move_pte() call in the original code before checking this:

Oh, maybe that's why.

Quoted 11 lines
		pte = get_and_clear_full_ptes(mm, old_addr, old_ptep, nr_ptes, 0);
		pte = move_pte(pte, old_addr, new_addr);
		pte = move_soft_dirty_pte(pte);

		if (need_clear_uffd_wp && pte_marker_uffd_wp(pte))
			pte_clear(mm, new_addr, new_ptep);

But maybe it's because there was a presumption move_pte() would like you know, move a PTE entry? Which it doesn't, it - only on SPARC - does a hook to flush the dcache.

I wish we could remove move_pte() completely. Or just notify about the move of a present pte ... because it doesn't ever modify the pte val.

So renaming it to "arch_notify_move_pte()" or sth. like that that is a void function might even be better.

... I think we could go one step further if we already have the folio: we could call it arch_notify_move_folio_pte(), and simplify the sparc implementation ...

Anyhow, the move_pte() cleanup can be done separately. Splitting the present from !present case here should probably be done as a cleanup upfront.

Signature · 3 lines
Cheers,

David / dhildenb
reviewer Reviewed-by on v4 2/2
#19 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

In reply to Dev Jain
Quoted 9 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

Quoted 7 lines

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

Signature · 1 line
Pedro
#20 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

In reply to Dev Jain
Quoted 34 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +6 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
  	return pte;
  }
 
+static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t pte, int max_nr)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	struct folio *folio;
+

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 188

Quoted 10 lines
mm/mremap.c +10 -0
+	if (max_nr == 1)
+		return 1;
+
+	folio = vm_normal_folio(vma, addr, pte);
+	if (!folio || !folio_test_large(folio))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+			       NULL, NULL);
+}

Dev, I think there is another bug hiding in here. That function ignores the writable bit, which is not what you need here, in particular for anonymous folios in some cases.

Later set_ptes() could end up marking ptes writable that were not writable before, which is bad (at least for anonymous folios, maybe also for pagecache folios).

I think you really must respect the writable bit through something like FPB_RESPECT_WRITE.

I patched out the "pte_batch_hint(ptep, pte) == 1" check we have upstream to make it reproduce on x86_64, but the following reproducer should likely reproduce on aarch64 without further kernel modifications.

# ./mremap
BUG: Memory modified

#define _GNU_SOURCE
#include <stdint.h>
#include <string.h>
#include <stdbool.h>
#include <x86intrin.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>
#include <errno.h>
#include <signal.h>
#include <fcntl.h>
#include <sys/wait.h>

static size_t pagesize;
static size_t thpsize;
static int pagemap_fd;

static uint64_t pagemap_get_entry(int fd, char *start)

{
        const unsigned long pfn = (unsigned long)start / getpagesize();
        uint64_t entry;
        int ret;

        ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry));
        if (ret != sizeof(entry)) {
                perror("reading pagemap failed");
                exit(-1);
        }
        return entry;
}

static bool pagemap_is_populated(int fd, char *start)

{
        return pagemap_get_entry(fd, start) & ((1ULL << 62) | (1ULL << 63));
}

unsigned long pagemap_get_pfn(int fd, char *start)

{
        uint64_t entry = pagemap_get_entry(fd, start);

        /* If present (63th bit), PFN is at bit 0 -- 54. */
        if (entry & (1ULL << 63))
                return entry & 0x007fffffffffffffull;
        return -1ul;
}

int main(void)

{
        char *mem, *mmap_mem, *tmp, *mremap_mem = MAP_FAILED;
        size_t size, mmap_size;
        int ret;

        pagesize = getpagesize();
        thpsize = 2 * 1024 * 1024ul;
        pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
        if (pagemap_fd < 0) {
                perror("opening pagemap failed");
                return -1;
        }

        /* For alignment purposes, we need twice the thp size. */
        mmap_size = 2 * thpsize;
        mmap_mem = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
                        MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
        if (mmap_mem == MAP_FAILED) {
                perror("mmap() failed");
                return -1;
        }

        /* We need a THP-aligned memory area. */
        mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));

        ret = madvise(mem, thpsize, MADV_HUGEPAGE);
        if (ret) {
                perror("MADV_HUGEPAGE failed");
                return -1;
        }

        /*
         * Try to populate a THP. Touch the first sub-page and test if we get
         * another sub-page populated automatically.
         */
        mem[0] = 0;
        if (!pagemap_is_populated(pagemap_fd, mem + pagesize)) {
                perror("Did not get a THP populated");
                return -1;
        }

        /* Share only the first page of the THP. */
        if (madvise(mem, pagesize, MADV_DONTFORK)) {
                perror("MADV_DONTFORK failed");
                return -1;
        }

        ret = fork();
        if (ret < 0) {
                perror("fork() failed");
                return -1;
        } else if (!ret) {
                while (true) {
                        char c = *((volatile char *)(mem + pagesize));

                        if (c) {
                                fprintf(stderr, "BUG: Memory modified\n");
                                exit(-2);
                        }
                }
        }

        /* Merge VMAs again. */
        if (madvise(mem, pagesize, MADV_DOFORK)) {
                perror("MADV_DONTFORK failed");
                return -1;
        }

        /* Mremap multiple pages. */
        mremap_mem = mmap(NULL, 2 * pagesize, PROT_NONE,
                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
        if (mem == MAP_FAILED) {
                perror("mmap() failed");
                return -1;
        }
        tmp = mremap(mem, 2 * pagesize, 2 * pagesize, MREMAP_MAYMOVE | MREMAP_FIXED,
                     mremap_mem);
        if (tmp != mremap_mem) {
                perror("mremap() failed");
                return -1;
        }

        /* Write into both pages. The child should never see these updates. */
        memset(mremap_mem, 1, 2 * pagesize);

        pause();
}
Signature · 3 lines
Cheers

David / dhildenb
author
#21 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Quoted 53 lines · David Hildenbrand
Quoted 33 lines · Dev Jain

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16 entries to collect a/d bits. Hence this optimization will result in a 16x reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other arches (including the general case), a minor improvement is expected due to a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------

  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +5 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
       return pte;
   }
   +static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr,
+        pte_t *ptep, pte_t pte, int max_nr)
+{
+    const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+    struct folio *folio;
+

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 187

Quoted 10 lines
mm/mremap.c +10 -0
+    if (max_nr == 1)
+        return 1;
+
+    folio = vm_normal_folio(vma, addr, pte);
+    if (!folio || !folio_test_large(folio))
+        return 1;
+
+    return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                   NULL, NULL);
+}

Dev, I think there is another bug hiding in here. That function ignores the writable bit, which is not what you need here, in particular for anonymous folios in some cases.

Later set_ptes() could end up marking ptes writable that were not writable before, which is bad (at least for anonymous folios, maybe also for pagecache folios).

Quoted 6 lines

I think you really must respect the writable bit through something like FPB_RESPECT_WRITE.

I patched out the "pte_batch_hint(ptep, pte) == 1" check we have upstream to make it reproduce on x86_64, but the following reproducer should likely reproduce on aarch64 without further kernel modifications.

You are right. Thanks! I recall during the mremap/mprotect stuff I had completely

forgotten that batching by default ignores the writable bit and remembered it during

the last version of mprotect series :(

Thanks for giving a reproducer; for some reason I am unable to reproduce on my machine as-is.

In any case the bug is obvious and I'll send out a patch.

Quoted 141 lines

# ./mremap
BUG: Memory modified

#define _GNU_SOURCE
#include <stdint.h>
#include <string.h>
#include <stdbool.h>
#include <x86intrin.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>
#include <errno.h>
#include <signal.h>
#include <fcntl.h>
#include <sys/wait.h>

static size_t pagesize;
static size_t thpsize;
static int pagemap_fd;

static uint64_t pagemap_get_entry(int fd, char *start)
{
        const unsigned long pfn = (unsigned long)start / getpagesize(); uint64_t entry;
        int ret;

        ret = pread(fd, &entry, sizeof(entry), pfn * sizeof(entry)); if (ret != sizeof(entry)) {
                perror("reading pagemap failed"); exit(-1);
        }
        return entry;
}

static bool pagemap_is_populated(int fd, char *start)
{
        return pagemap_get_entry(fd, start) & ((1ULL << 62) | (1ULL << 63)); }

unsigned long pagemap_get_pfn(int fd, char *start) {
        uint64_t entry = pagemap_get_entry(fd, start);

        /* If present (63th bit), PFN is at bit 0 -- 54. */ if (entry & (1ULL << 63))
                return entry & 0x007fffffffffffffull; return -1ul;
}

int main(void)
{
        char *mem, *mmap_mem, *tmp, *mremap_mem = MAP_FAILED; size_t size, mmap_size;
        int ret;

        pagesize = getpagesize();
        thpsize = 2 * 1024 * 1024ul;
        pagemap_fd = open("/proc/self/pagemap", O_RDONLY); if (pagemap_fd < 0) {
                perror("opening pagemap failed"); return -1;
        }

        /* For alignment purposes, we need twice the thp size. */ mmap_size = 2 * thpsize;
        mmap_mem = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (mmap_mem == MAP_FAILED) {
                perror("mmap() failed");
                return -1;
        }

        /* We need a THP-aligned memory area. */
        mem = (char *)(((uintptr_t)mmap_mem + thpsize) & ~(thpsize - 1));

        ret = madvise(mem, thpsize, MADV_HUGEPAGE); if (ret) {
                perror("MADV_HUGEPAGE failed"); return -1;
        }

        /*
         * Try to populate a THP. Touch the first sub-page and test if we get
         * another sub-page populated automatically.
         */
        mem[0] = 0;
        if (!pagemap_is_populated(pagemap_fd, mem + pagesize)) { perror("Did not get a THP populated");
                return -1;
        }

        /* Share only the first page of the THP. */ if (madvise(mem, pagesize, MADV_DONTFORK)) { perror("MADV_DONTFORK failed"); return -1;
        }

        ret = fork();
        if (ret < 0) {
                perror("fork() failed");
                return -1;
        } else if (!ret) {
                while (true) {
                        char c = *((volatile char *)(mem + pagesize));

                        if (c) {
                                fprintf(stderr, "BUG: Memory modified\n"); exit(-2);
                        }
                }
        }

        /* Merge VMAs again. */
        if (madvise(mem, pagesize, MADV_DOFORK)) { perror("MADV_DONTFORK failed"); return -1;
        }

        /* Mremap multiple pages. */
        mremap_mem = mmap(NULL, 2 * pagesize, PROT_NONE,
                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
        if (mem == MAP_FAILED) {
                perror("mmap() failed");
                return -1;
        }
        tmp = mremap(mem, 2 * pagesize, 2 * pagesize, MREMAP_MAYMOVE | MREMAP_FIXED, mremap_mem);
        if (tmp != mremap_mem) {
                perror("mremap() failed");
                return -1;
        }

        /* Write into both pages. The child should never see these updates. */ memset(mremap_mem, 1, 2 * pagesize);

        pause();
}

#22 lore raw

Re: [PATCH v4 2/2] mm: Optimize mremap() by PTE batching

Dev Jain <[email protected]> schrieb am Di. 28. Okt. 2025 um 06:32:

Quoted 74 lines · David Hildenbrand
Quoted 61 lines · Dev Jain
Quoted 39 lines

Use folio_pte_batch() to optimize move_ptes(). On arm64, if the ptes are painted with the contig bit, then ptep_get() will iterate through all 16
entries to collect a/d bits. Hence this optimization will result in a 16x
reduction in the number of ptep_get() calls. Next, ptep_get_and_clear() will eventually call contpte_try_unfold() on every contig block, thus flushing the TLB for the complete large folio range. Instead, use get_and_clear_full_ptes() so as to elide TLBIs on each contig block, and only
do them on the starting and ending contig block.

For split folios, there will be no pte batching; nr_ptes will be 1. For pagetable splitting, the ptes will still point to the same large folio; for arm64, this results in the optimization described above, and for other
arches (including the general case), a minor improvement is expected due to
a reduction in the number of function calls.

---
  mm/mremap.c | 39 ++++++++++++++++++++++++++++++++-------
  1 file changed, 32 insertions(+), 7 deletions(-)
mm/mremap.c +5 -0
index 180b12225368..18b215521ada 100644
@@ -170,6 +170,23 @@ static pte_t move_soft_dirty_pte(pte_t pte)
      return pte;
  }
  +static int mremap_folio_pte_batch(struct vm_area_struct *vma,
 unsigned long addr,
+        pte_t *ptep, pte_t pte, int max_nr)
+{
+    const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+    struct folio *folio;
+

mm/mremap.c static pte_t move_soft_dirty_pte(pte_t pte) · line 188

Quoted 10 lines
mm/mremap.c +10 -0
+    if (max_nr == 1)
+        return 1;
+
+    folio = vm_normal_folio(vma, addr, pte);
+    if (!folio || !folio_test_large(folio))
+        return 1;
+
+    return folio_pte_batch(folio, addr, ptep, pte, max_nr, flags, NULL,
+                   NULL, NULL);
+}

Dev, I think there is another bug hiding in here. That function ignores the writable bit, which is not what you need here, in particular for anonymous folios in some cases.

Later set_ptes() could end up marking ptes writable that were not writable
before, which is bad (at least for anonymous folios, maybe also for pagecache
folios).

Quoted 7 lines

I think you really must respect the writable bit through something like FPB_RESPECT_WRITE.

I patched out the "pte_batch_hint(ptep, pte) == 1" check we have upstream to make it reproduce on x86_64, but the following reproducer should likely
reproduce on aarch64 without further kernel modifications.

You are right. Thanks! I recall during the mremap/mprotect stuff I had completely

Quoted 8 lines

forgotten that batching by default ignores the writable bit and remembered it during

the last version of mprotect series :(

Thanks for giving a reproducer; for some reason I am unable to reproduce on my machine as-is.

(Writing from gmail app, probably it will mess up html)

I think leaving only a single page unshared is not sufficient. Try with 16, such that they will be pte-cont and make the initial hint check happy.

Quoted 1 line
#23 lore raw
In reply to Dev Jain
Quoted 7 lines · Dev Jain

Currently move_ptes() iterates through ptes one by one. If the underlying folio mapped by the ptes is large, we can process those ptes in a batch using folio_pte_batch(), thus clearing and setting the PTEs in one go. For arm64 specifically, this results in a 16x reduction in the number of ptep_get() calls (since on a contig block, ptep_get() on arm64 will iterate through all 16 entries to collect a/d bits), and we also elide extra TLBIs through get_and_clear_full_ptes, replacing ptep_get_and_clear.

Thanks this is good!

Quoted 5 lines

Mapping 1M of memory with 64K folios, memsetting it, remapping it to src + 1M, and munmapping it 10,000 times, the average execution time reduces from 1.9 to 1.2 seconds, giving a 37% performance optimization, on Apple M3 (arm64). No regression is observed for small folios.

Hmm, I thought people were struggling to get M3 to work with Asahi? :) or is this in a mac-based vm? I've not paid attention to recent developments.

Quoted 35 lines

The patchset is based on mm-unstable (6ebffe676fcf).

Test program for reference:

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include <errno.h>

#define SIZE (1UL << 20) // 1M

int main(void) {
    void *new_addr, *addr;

    for (int i = 0; i < 10000; ++i) {
        addr = mmap((void *)(1UL << 30), SIZE, PROT_READ | PROT_WRITE,
                    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
        if (addr == MAP_FAILED) {
                perror("mmap");
                return 1;
        }
        memset(addr, 0xAA, SIZE);

        new_addr = mremap(addr, SIZE, SIZE, MREMAP_MAYMOVE | MREMAP_FIXED, addr + SIZE);
        if (new_addr != (addr + SIZE)) {
                perror("mremap");
                return 1;
        }
        munmap(new_addr, SIZE);
    }
Quoted 2 lines

}

Thanks for including! Very useful.

Quoted 6 lines

v3->v4:
- Remove comment above mremap_folio_pte_batch, improve patch description differentiating between folio splitting and pagetable splitting
v2->v3:
- Refactor mremap_folio_pte_batch, drop maybe_contiguous_pte_pfns, fix indentation (Lorenzo), fix cover letter description (512K -> 1M)

It's nitty but these seem to be getting more and more abbreviated :) not a massive big deal however ;)

Quoted 18 lines

v1->v2:
- Expand patch descriptions, move pte declarations to a new line, reduce indentation in patch 2 by introducing mremap_folio_pte_batch(), fix loop iteration (Lorenzo)
- Merge patch 2 and 3 (Anshuman, Lorenzo)
- Fix maybe_contiguous_pte_pfns (Willy)

Dev Jain (2):
mm: Call pointers to ptes as ptep
mm: Optimize mremap() by PTE batching

 mm/mremap.c | 58 ++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 16 deletions(-)

--
2.30.2

author
#24 lore raw
Quoted 8 lines · Lorenzo Stoakes
Quoted 7 lines · Dev Jain

Currently move_ptes() iterates through ptes one by one. If the underlying folio mapped by the ptes is large, we can process those ptes in a batch using folio_pte_batch(), thus clearing and setting the PTEs in one go. For arm64 specifically, this results in a 16x reduction in the number of ptep_get() calls (since on a contig block, ptep_get() on arm64 will iterate through all 16 entries to collect a/d bits), and we also elide extra TLBIs through get_and_clear_full_ptes, replacing ptep_get_and_clear.

Quoted 8 lines

Thanks this is good!

Quoted 4 lines

Mapping 1M of memory with 64K folios, memsetting it, remapping it to src + 1M, and munmapping it 10,000 times, the average execution time reduces from 1.9 to 1.2 seconds, giving a 37% performance optimization, on Apple M3 (arm64). No regression is observed for small folios.

Hmm, I thought people were struggling to get M3 to work with Asahi? :) or is this in a mac-based vm? I've not paid attention to recent developments.

I meant a Linux VM on Mac.

Quoted 65 lines
Quoted 35 lines

The patchset is based on mm-unstable (6ebffe676fcf).

Test program for reference:

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <string.h>
#include <errno.h>

#define SIZE (1UL << 20) // 1M

int main(void) {
     void *new_addr, *addr;

     for (int i = 0; i < 10000; ++i) {
         addr = mmap((void *)(1UL << 30), SIZE, PROT_READ | PROT_WRITE,
                     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
         if (addr == MAP_FAILED) {
                 perror("mmap");
                 return 1;
         }
         memset(addr, 0xAA, SIZE);

         new_addr = mremap(addr, SIZE, SIZE, MREMAP_MAYMOVE | MREMAP_FIXED, addr + SIZE);
         if (new_addr != (addr + SIZE)) {
                 perror("mremap");
                 return 1;
         }
         munmap(new_addr, SIZE);
     }
Quoted 2 lines

}

Thanks for including! Very useful.

Quoted 6 lines

v3->v4:
- Remove comment above mremap_folio_pte_batch, improve patch description differentiating between folio splitting and pagetable splitting
v2->v3:
- Refactor mremap_folio_pte_batch, drop maybe_contiguous_pte_pfns, fix

    indentation (Lorenzo), fix cover letter description (512K -> 1M)

It's nitty but these seem to be getting more and more abbreviated :) not a massive big deal however ;)

Quoted 17 lines

v1->v2:
- Expand patch descriptions, move pte declarations to a new line, reduce indentation in patch 2 by introducing mremap_folio_pte_batch(), fix loop iteration (Lorenzo)
- Merge patch 2 and 3 (Anshuman, Lorenzo)
- Fix maybe_contiguous_pte_pfns (Willy)

Dev Jain (2):
mm: Call pointers to ptes as ptep
mm: Optimize mremap() by PTE batching

  mm/mremap.c | 58 ++++++++++++++++++++++++++++++++++++++---------------
  1 file changed, 42 insertions(+), 16 deletions(-)

--
2.30.2