Skip to contentSkip to conversation

Optimize mprotect() for large folios

PATCH Superseded Dev Jain 4 participants 34 messages May 19, 2025 – Jun 27, 2025

Subsystems (4)

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Dev Jain

author
#0 lore raw

v3 cover Optimize mprotect() for large folios

This patchset optimizes the mprotect() system call for large folios by PTE-batching.

We use the following test cases to measure performance, mprotect()'ing the mapped memory to read-only then read-write 40 times:

Test case 1: Mapping 1G of memory, touching it to get PMD-THPs, then pte-mapping those THPs
Test case 2: Mapping 1G of memory with 64K mTHPs
Test case 3: Mapping 1G of memory with 4K pages

Average execution time on arm64, Apple M3:
Before the patchset:
T1: 7.9 seconds T2: 7.9 seconds T3: 4.2 seconds

After the patchset:
T1: 2.1 seconds T2: 2.2 seconds T3: 4.3 seconds

Observing T1/T2 and T3 before the patchset, we also remove the regression introduced by ptep_get() on a contpte block. And, for large folios we get an almost 74% performance improvement, albeit the trade-off being a slight degradation in the small folio case.

Here is the test program:

#define _GNU_SOURCE
#include <sys/mman.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>

#define SIZE (1024*1024*1024)

unsigned long pmdsize = (1UL << 21);
unsigned long pagesize = (1UL << 12);

static void pte_map_thps(char *mem, size_t size)

{
	size_t offs;
	int ret = 0;


	/* PTE-map each THP by temporarily splitting the VMAs. */
	for (offs = 0; offs < size; offs += pmdsize) {
		ret |= madvise(mem + offs, pagesize, MADV_DONTFORK);
		ret |= madvise(mem + offs, pagesize, MADV_DOFORK);
	}

	if (ret) {
		fprintf(stderr, "ERROR: mprotect() failed\n");
		exit(1);
	}
}

int main(int argc, char *argv[])

{
	char *p;
        int ret = 0;
	p = mmap((1UL << 30), SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
	if (p != (1UL << 30)) {
		perror("mmap");
		return 1;
	}



	memset(p, 0, SIZE);
	if (madvise(p, SIZE, MADV_NOHUGEPAGE))
		perror("madvise");
	explicit_bzero(p, SIZE);
	pte_map_thps(p, SIZE);

	for (int loops = 0; loops < 40; loops++) {
		if (mprotect(p, SIZE, PROT_READ))
			perror("mprotect"), exit(1);
		if (mprotect(p, SIZE, PROT_READ|PROT_WRITE))
			perror("mprotect"), exit(1);
		explicit_bzero(p, SIZE);
	}
}

The patchset is rebased onto mm-unstable (9ead4336d7c07f085def6ab372245640a22af5bd).

v2->v3:
- Add comments for the new APIs (Ryan, Lorenzo)
- Instead of refactoring, use a "skip_batch" label
- Move arm64 patches at the end (Ryan)
- In can_change_pte_writable(), check AnonExclusive page-by-page (David H)
- Resolve implicit declaration; tested build on x86 (Lance Yang)

v1->v2:
- Rebase onto mm-unstable (6ebffe676fcf: util_macros.h: make the header more resilient)
- Abridge the anon-exclusive condition (Lance Yang)

Dev Jain (5):
mm: Optimize mprotect() by batch-skipping PTEs
mm: Add batched versions of ptep_modify_prot_start/commit
mm: Optimize mprotect() by PTE batching
arm64: Add batched version of ptep_modify_prot_start
arm64: Add batched version of ptep_modify_prot_commit

 arch/arm64/include/asm/pgtable.h |  10 ++
 arch/arm64/mm/mmu.c              |  21 ++++-
 include/linux/mm.h               |   7 +-
 include/linux/pgtable.h          |  79 ++++++++++++++++
 mm/mprotect.c                    | 154 +++++++++++++++++++++++++------
 mm/pgtable-generic.c             |  16 +++-
 6 files changed, 246 insertions(+), 41 deletions(-)
Signature · 1 line
2.30.2
author
#1 lore raw

PATCH v3 1/5 mm: Optimize mprotect() by batch-skipping PTEs

In case of prot_numa, there are various cases in which we can skip to the next iteration. Since the skip condition is based on the folio and not the PTEs, we can skip a PTE batch.

---
 mm/mprotect.c | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)
mm/mprotect.c +29 -7
index 88608d0dc2c2..1ee160ed0b14 100644
@@ -83,6 +83,18 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 	return pte_dirty(pte);
 }
 
+static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
+		pte_t pte, int max_nr_ptes)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
+			       NULL, NULL, NULL);
+}
+
 static long change_pte_range(struct mmu_gather *tlb,
 		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
 		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -94,6 +106,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	int nr_ptes;
 
 	tlb_change_page_size(tlb, PAGE_SIZE);
 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -108,8 +121,10 @@ static long change_pte_range(struct mmu_gather *tlb,
 	flush_tlb_batched_pending(vma->vm_mm);
 	arch_enter_lazy_mmu_mode();
 	do {
+		nr_ptes = 1;
 		oldpte = ptep_get(pte);
 		if (pte_present(oldpte)) {
+			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
 			pte_t ptent;
 
 			/*
@@ -126,15 +141,18 @@ static long change_pte_range(struct mmu_gather *tlb,
 					continue;
 
 				folio = vm_normal_folio(vma, addr, oldpte);
-				if (!folio || folio_is_zone_device(folio) ||
-				    folio_test_ksm(folio))
+				if (!folio)
 					continue;
 
+				if (folio_is_zone_device(folio) ||
+				    folio_test_ksm(folio))
+					goto skip_batch;
+
 				/* Also skip shared copy-on-write pages */
 				if (is_cow_mapping(vma->vm_flags) &&
 				    (folio_maybe_dma_pinned(folio) ||
 				     folio_maybe_mapped_shared(folio)))
-					continue;
+					goto skip_batch;
 
 				/*
 				 * While migration can move some dirty pages,
@@ -143,7 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 */
 				if (folio_is_file_lru(folio) &&
 				    folio_test_dirty(folio))
-					continue;
+					goto skip_batch;
 
 				/*
 				 * Don't mess with PTEs if page is already on the node
@@ -151,7 +169,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 */
 				nid = folio_nid(folio);
 				if (target_node == nid)
-					continue;
+					goto skip_batch;
 				toptier = node_is_toptier(nid);
 
 				/*
@@ -159,8 +177,12 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 * balancing is disabled
 				 */
 				if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
-				    toptier)
+				    toptier) {
+skip_batch:
+					nr_ptes = mprotect_batch(folio, addr, pte,
+								 oldpte, max_nr_ptes);
 					continue;
+				}
 				if (folio_use_access_time(folio))
 					folio_xchg_access_time(folio,
 						jiffies_to_msecs(jiffies));
@@ -280,7 +302,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				pages++;
 			}
 		}
-	} while (pte++, addr += PAGE_SIZE, addr != end);
+	} while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
 	arch_leave_lazy_mmu_mode();
 	pte_unmap_unlock(pte - 1, ptl);
 
Signature · 1 line
2.30.2
Reviewed-by on v3 1/5
#2 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 5 lines · Dev Jain

In case of prot_numa, there are various cases in which we can skip to the next iteration. Since the skip condition is based on the folio and not the PTEs, we can skip a PTE batch.

LGTM; a lot less churn than before.

Quoted 109 lines
---
 mm/mprotect.c | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)
mm/mprotect.c +29 -7
index 88608d0dc2c2..1ee160ed0b14 100644
@@ -83,6 +83,18 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 	return pte_dirty(pte);
 }
 
+static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
+		pte_t pte, int max_nr_ptes)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
+			       NULL, NULL, NULL);
+}
+
 static long change_pte_range(struct mmu_gather *tlb,
 		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
 		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -94,6 +106,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	int nr_ptes;
 
 	tlb_change_page_size(tlb, PAGE_SIZE);
 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -108,8 +121,10 @@ static long change_pte_range(struct mmu_gather *tlb,
 	flush_tlb_batched_pending(vma->vm_mm);
 	arch_enter_lazy_mmu_mode();
 	do {
+		nr_ptes = 1;
 		oldpte = ptep_get(pte);
 		if (pte_present(oldpte)) {
+			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
 			pte_t ptent;
 
 			/*
@@ -126,15 +141,18 @@ static long change_pte_range(struct mmu_gather *tlb,
 					continue;
 
 				folio = vm_normal_folio(vma, addr, oldpte);
-				if (!folio || folio_is_zone_device(folio) ||
-				    folio_test_ksm(folio))
+				if (!folio)
 					continue;
 
+				if (folio_is_zone_device(folio) ||
+				    folio_test_ksm(folio))
+					goto skip_batch;
+
 				/* Also skip shared copy-on-write pages */
 				if (is_cow_mapping(vma->vm_flags) &&
 				    (folio_maybe_dma_pinned(folio) ||
 				     folio_maybe_mapped_shared(folio)))
-					continue;
+					goto skip_batch;
 
 				/*
 				 * While migration can move some dirty pages,
@@ -143,7 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 */
 				if (folio_is_file_lru(folio) &&
 				    folio_test_dirty(folio))
-					continue;
+					goto skip_batch;
 
 				/*
 				 * Don't mess with PTEs if page is already on the node
@@ -151,7 +169,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 */
 				nid = folio_nid(folio);
 				if (target_node == nid)
-					continue;
+					goto skip_batch;
 				toptier = node_is_toptier(nid);
 
 				/*
@@ -159,8 +177,12 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 * balancing is disabled
 				 */
 				if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
-				    toptier)
+				    toptier) {
+skip_batch:
+					nr_ptes = mprotect_batch(folio, addr, pte,
+								 oldpte, max_nr_ptes);
 					continue;
+				}
 				if (folio_use_access_time(folio))
 					folio_xchg_access_time(folio,
 						jiffies_to_msecs(jiffies));
@@ -280,7 +302,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				pages++;
 			}
 		}
-	} while (pte++, addr += PAGE_SIZE, addr != end);
+	} while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
 	arch_leave_lazy_mmu_mode();
 	pte_unmap_unlock(pte - 1, ptl);
 
#3 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

In reply to Dev Jain
Quoted 14 lines · Dev Jain

In case of prot_numa, there are various cases in which we can skip to the next iteration. Since the skip condition is based on the folio and not the PTEs, we can skip a PTE batch.

---
 mm/mprotect.c | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)
mm/mprotect.c +0 -0
index 88608d0dc2c2..1ee160ed0b14 100644

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 86

Quoted 4 lines
mm/mprotect.c +1 -0
@@ -83,6 +83,18 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 	return pte_dirty(pte);
 }
 
+static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,

Perhaps it should be called mprotect_folio_pte_batch() to match the existing madvise_folio_pte_batch()?

Quoted 34 lines
mm/mprotect.c +14 -0
+		pte_t pte, int max_nr_ptes)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
+			       NULL, NULL, NULL);
+}
+
 static long change_pte_range(struct mmu_gather *tlb,
 		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
 		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -94,6 +106,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	int nr_ptes;
 
 	tlb_change_page_size(tlb, PAGE_SIZE);
 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -108,8 +121,10 @@ static long change_pte_range(struct mmu_gather *tlb,
 	flush_tlb_batched_pending(vma->vm_mm);
 	arch_enter_lazy_mmu_mode();
 	do {
+		nr_ptes = 1;
 		oldpte = ptep_get(pte);
 		if (pte_present(oldpte)) {
+			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
 			pte_t ptent;
 
 			/*

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 145

Quoted 7 lines
mm/mprotect.c +1 -2
@@ -126,15 +141,18 @@ static long change_pte_range(struct mmu_gather *tlb,
 					continue;
 
 				folio = vm_normal_folio(vma, addr, oldpte);
-				if (!folio || folio_is_zone_device(folio) ||
-				    folio_test_ksm(folio))
+				if (!folio)
 					continue;

You modify mprotect_batch() to handle folio == NULL later, perhaps just add that here, then you don't need to unpick this conditional and can just goto skip_branch, even for the !folio case.

Thanks,
Ryan

Quoted 55 lines
mm/mprotect.c +13 -5
+				if (folio_is_zone_device(folio) ||
+				    folio_test_ksm(folio))
+					goto skip_batch;
+
 				/* Also skip shared copy-on-write pages */
 				if (is_cow_mapping(vma->vm_flags) &&
 				    (folio_maybe_dma_pinned(folio) ||
 				     folio_maybe_mapped_shared(folio)))
-					continue;
+					goto skip_batch;
 
 				/*
 				 * While migration can move some dirty pages,
@@ -143,7 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 */
 				if (folio_is_file_lru(folio) &&
 				    folio_test_dirty(folio))
-					continue;
+					goto skip_batch;
 
 				/*
 				 * Don't mess with PTEs if page is already on the node
@@ -151,7 +169,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 */
 				nid = folio_nid(folio);
 				if (target_node == nid)
-					continue;
+					goto skip_batch;
 				toptier = node_is_toptier(nid);
 
 				/*
@@ -159,8 +177,12 @@ static long change_pte_range(struct mmu_gather *tlb,
 				 * balancing is disabled
 				 */
 				if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
-				    toptier)
+				    toptier) {
+skip_batch:
+					nr_ptes = mprotect_batch(folio, addr, pte,
+								 oldpte, max_nr_ptes);
 					continue;
+				}
 				if (folio_use_access_time(folio))
 					folio_xchg_access_time(folio,
 						jiffies_to_msecs(jiffies));
@@ -280,7 +302,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 				pages++;
 			}
 		}
-	} while (pte++, addr += PAGE_SIZE, addr != end);
+	} while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
 	arch_leave_lazy_mmu_mode();
 	pte_unmap_unlock(pte - 1, ptl);
 
author
#4 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 18 lines · Ryan Roberts
Quoted 17 lines · Dev Jain

In case of prot_numa, there are various cases in which we can skip to the next iteration. Since the skip condition is based on the folio and not the PTEs, we can skip a PTE batch.

---
  mm/mprotect.c | 36 +++++++++++++++++++++++++++++-------
  1 file changed, 29 insertions(+), 7 deletions(-)
mm/mprotect.c +0 -0
index 88608d0dc2c2..1ee160ed0b14 100644
@@ -83,6 +83,18 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  	return pte_dirty(pte);
  }
 
Quoted 3 lines

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 86

Quoted 1 line
mm/mprotect.c +1 -0
+static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,

Perhaps it should be called mprotect_folio_pte_batch() to match the existing madvise_folio_pte_batch()?

Thanks, this is better.

Quoted 37 lines
Quoted 37 lines
mm/mprotect.c +14 -0
+		pte_t pte, int max_nr_ptes)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
+			       NULL, NULL, NULL);
+}
+
  static long change_pte_range(struct mmu_gather *tlb,
  		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
  		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -94,6 +106,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
  	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
  	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	int nr_ptes;
 
  	tlb_change_page_size(tlb, PAGE_SIZE);
  	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -108,8 +121,10 @@ static long change_pte_range(struct mmu_gather *tlb,
  	flush_tlb_batched_pending(vma->vm_mm);
  	arch_enter_lazy_mmu_mode();
  	do {
+		nr_ptes = 1;
  		oldpte = ptep_get(pte);
  		if (pte_present(oldpte)) {
+			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
  			pte_t ptent;
 
  			/*
@@ -126,15 +141,18 @@ static long change_pte_range(struct mmu_gather *tlb,
  					continue;
 
Quoted 8 lines

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 145

Quoted 5 lines
mm/mprotect.c +1 -2
  				folio = vm_normal_folio(vma, addr, oldpte);
-				if (!folio || folio_is_zone_device(folio) ||
-				    folio_test_ksm(folio))
+				if (!folio)
  					continue;

You modify mprotect_batch() to handle folio == NULL later, perhaps just add that here, then you don't need to unpick this conditional and can just goto skip_branch, even for the !folio case.

I'll check this.

Quoted 59 lines

Thanks,
Ryan

Quoted 55 lines
mm/mprotect.c +13 -5
+				if (folio_is_zone_device(folio) ||
+				    folio_test_ksm(folio))
+					goto skip_batch;
+
  				/* Also skip shared copy-on-write pages */
  				if (is_cow_mapping(vma->vm_flags) &&
  				    (folio_maybe_dma_pinned(folio) ||
  				     folio_maybe_mapped_shared(folio)))
-					continue;
+					goto skip_batch;
 
  				/*
  				 * While migration can move some dirty pages,
@@ -143,7 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  				 */
  				if (folio_is_file_lru(folio) &&
  				    folio_test_dirty(folio))
-					continue;
+					goto skip_batch;
 
  				/*
  				 * Don't mess with PTEs if page is already on the node
@@ -151,7 +169,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  				 */
  				nid = folio_nid(folio);
  				if (target_node == nid)
-					continue;
+					goto skip_batch;
  				toptier = node_is_toptier(nid);
 
  				/*
@@ -159,8 +177,12 @@ static long change_pte_range(struct mmu_gather *tlb,
  				 * balancing is disabled
  				 */
  				if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
-				    toptier)
+				    toptier) {
+skip_batch:
+					nr_ptes = mprotect_batch(folio, addr, pte,
+								 oldpte, max_nr_ptes);
  					continue;
+				}
  				if (folio_use_access_time(folio))
  					folio_xchg_access_time(folio,
  						jiffies_to_msecs(jiffies));
@@ -280,7 +302,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  				pages++;
  			}
  		}
-	} while (pte++, addr += PAGE_SIZE, addr != end);
+	} while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end);
  	arch_leave_lazy_mmu_mode();
  	pte_unmap_unlock(pte - 1, ptl);
 
#5 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

In reply to Dev Jain

On 19.05.25 09:48, Dev Jain wrote:

Please highlight in the subject that this is only about MM_CP_PROT_NUMA handling.

Quoted 92 lines

In case of prot_numa, there are various cases in which we can skip to the next iteration. Since the skip condition is based on the folio and not the PTEs, we can skip a PTE batch.

---
  mm/mprotect.c | 36 +++++++++++++++++++++++++++++-------
  1 file changed, 29 insertions(+), 7 deletions(-)
mm/mprotect.c +23 -5
index 88608d0dc2c2..1ee160ed0b14 100644
@@ -83,6 +83,18 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  	return pte_dirty(pte);
  }
 
+static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
+		pte_t pte, int max_nr_ptes)
+{
+	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+		return 1;
+
+	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
+			       NULL, NULL, NULL);
+}
+
  static long change_pte_range(struct mmu_gather *tlb,
  		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
  		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -94,6 +106,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
  	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
  	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+	int nr_ptes;
 
  	tlb_change_page_size(tlb, PAGE_SIZE);
  	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -108,8 +121,10 @@ static long change_pte_range(struct mmu_gather *tlb,
  	flush_tlb_batched_pending(vma->vm_mm);
  	arch_enter_lazy_mmu_mode();
  	do {
+		nr_ptes = 1;
  		oldpte = ptep_get(pte);
  		if (pte_present(oldpte)) {
+			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
  			pte_t ptent;
 
  			/*
@@ -126,15 +141,18 @@ static long change_pte_range(struct mmu_gather *tlb,
  					continue;
 
  				folio = vm_normal_folio(vma, addr, oldpte);
-				if (!folio || folio_is_zone_device(folio) ||
-				    folio_test_ksm(folio))
+				if (!folio)
  					continue;
 
+				if (folio_is_zone_device(folio) ||
+				    folio_test_ksm(folio))
+					goto skip_batch;
+
  				/* Also skip shared copy-on-write pages */
  				if (is_cow_mapping(vma->vm_flags) &&
  				    (folio_maybe_dma_pinned(folio) ||
  				     folio_maybe_mapped_shared(folio)))
-					continue;
+					goto skip_batch;
 
  				/*
  				 * While migration can move some dirty pages,
@@ -143,7 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  				 */
  				if (folio_is_file_lru(folio) &&
  				    folio_test_dirty(folio))
-					continue;
+					goto skip_batch;
 
  				/*
  				 * Don't mess with PTEs if page is already on the node
@@ -151,7 +169,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  				 */
  				nid = folio_nid(folio);
  				if (target_node == nid)
-					continue;
+					goto skip_batch;
  				toptier = node_is_toptier(nid);
 
  				/*

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 185

Quoted 10 lines
mm/mprotect.c +5 -1
@@ -159,8 +177,12 @@ static long change_pte_range(struct mmu_gather *tlb,
  				 * balancing is disabled
  				 */
  				if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) &&
-				    toptier)
+				    toptier) {
+skip_batch:
+					nr_ptes = mprotect_batch(folio, addr, pte,
+								 oldpte, max_nr_ptes);
  					continue;
+				}

I suggest

a) not burying that skip_batch label in another if condition

b) looking into factoring out prot_numa handling into a separate function first.

Likely we want something like

if (prot_numa) {
	nr_ptes = prot_numa_pte_range_skip_ptes(vma, addr, oldpte);
	if (nr_ptes)
		continue;
}

... likely with a better function name,

Signature · 3 lines
Cheers,

David / dhildenb
author
#6 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 3 lines · David Hildenbrand

On 19.05.25 09:48, Dev Jain wrote:

Please highlight in the subject that this is only about MM_CP_PROT_NUMA handling.

Sure.

Quoted 110 lines
Quoted 91 lines

In case of prot_numa, there are various cases in which we can skip to the next iteration. Since the skip condition is based on the folio and not the PTEs, we can skip a PTE batch.

---
  mm/mprotect.c | 36 +++++++++++++++++++++++++++++-------

  1 file changed, 29 insertions(+), 7 deletions(-)
mm/mprotect.c +11 -0
index 88608d0dc2c2..1ee160ed0b14 100644
@@ -83,6 +83,18 @@ bool can_change_pte_writable(struct vm_area_struct 
 *vma, unsigned long addr,
       return pte_dirty(pte);
   }
   +static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
+        pte_t pte, int max_nr_ptes)
+{
+    const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+
+    if (!folio_test_large(folio) || (max_nr_ptes == 1))
+        return 1;
+
+    return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
+                   NULL, NULL, NULL);
+}
+
   static long change_pte_range(struct mmu_gather *tlb,
           struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,

          unsigned long end, pgprot_t newprot, unsigned long cp_flags)

mm/mprotect.c +4 -2
@@ -94,6 +106,7 @@ static long change_pte_range(struct mmu_gather *tlb,
       bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
       bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
       bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
+    int nr_ptes;
         tlb_change_page_size(tlb, PAGE_SIZE);
       pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -108,8 +121,10 @@ static long change_pte_range(struct mmu_gather 
 *tlb,
       flush_tlb_batched_pending(vma->vm_mm);
       arch_enter_lazy_mmu_mode();
       do {
+        nr_ptes = 1;
           oldpte = ptep_get(pte);
           if (pte_present(oldpte)) {
+            int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
               pte_t ptent;
                 /*
@@ -126,15 +141,18 @@ static long change_pte_range(struct mmu_gather 
 *tlb,
                       continue;
                     folio = vm_normal_folio(vma, addr, oldpte);
-                if (!folio || folio_is_zone_device(folio) ||
-                    folio_test_ksm(folio))
+                if (!folio)
                       continue;

  +                if (folio_is_zone_device(folio) || +                    folio_test_ksm(folio))
+                    goto skip_batch;
+
                  /* Also skip shared copy-on-write pages */ if (is_cow_mapping(vma->vm_flags) && (folio_maybe_dma_pinned(folio) || folio_maybe_mapped_shared(folio))) -                    continue;
+                    goto skip_batch;
                    /*
                   * While migration can move some dirty pages,

mm/mprotect.c +0 -0
@@ -143,7 +161,7 @@ static long change_pte_range(struct mmu_gather *tlb,
                    */

                  if (folio_is_file_lru(folio) && folio_test_dirty(folio))
-                    continue;
+                    goto skip_batch;
                    /*
                   * Don't mess with PTEs if page is already on the node

mm/mprotect.c +1 -1
@@ -151,7 +169,7 @@ static long change_pte_range(struct mmu_gather *tlb,
                    */
                   nid = folio_nid(folio);
                   if (target_node == nid)
-                    continue;
+                    goto skip_batch;
                   toptier = node_is_toptier(nid);
                     /*
@@ -159,8 +177,12 @@ static long change_pte_range(struct mmu_gather 
 *tlb,
                    * balancing is disabled
                    */
Quoted 8 lines

                  if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_NORMAL) && -                    toptier)
+                    toptier) {
+skip_batch:
+                    nr_ptes = mprotect_batch(folio, addr, pte,
+                                 oldpte, max_nr_ptes);
                      continue;
+                }

I suggest

a) not burying that skip_batch label in another if condition

b) looking into factoring out prot_numa handling into a separate function first.

Likely we want something like

Quoted 7 lines

if (prot_numa) {
    nr_ptes = prot_numa_pte_range_skip_ptes(vma, addr, oldpte); if (nr_ptes)
        continue;
}

... likely with a better function name,

I want to be able to reuse the folio from vm_normal_folio(), and we also need

nr_ptes to know how much to skip, so if there is no objection in passing int *nr_ptes,

or struct folio **foliop to this new function, then I'll carry on with your suggestion :)

Quoted 2 lines
#7 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 4 lines
Quoted 2 lines

... likely with a better function name,

Quoted 8 lines

I want to be able to reuse the folio from vm_normal_folio(), and we also need

nr_ptes to know how much to skip, so if there is no objection in passing int *nr_ptes,

or struct folio **foliop to this new function, then I'll carry on with your suggestion :)

Can you quickly prototype what you have in mind and paste it here? Will make it easier :)

Signature · 3 lines
Cheers,

David / dhildenb
author
#8 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 7 lines · David Hildenbrand
Quoted 7 lines
Quoted 2 lines

... likely with a better function name,

I want to be able to reuse the folio from vm_normal_folio(), and we also need

Quoted 7 lines
Quoted 5 lines

nr_ptes to know how much to skip, so if there is no objection in passing int *nr_ptes,

or struct folio **foliop to this new function, then I'll carry on with your suggestion :)

Can you quickly prototype what you have in mind and paste it here? Will make it easier :)

if (prot_numa)

    func(vma, addr, oldpte, &nr);

struct folio *folio func(vma, addr, oldpte, int *nr)

{

    if (pte_protnone(oldpte))

        *nr = 1, return NULL;

    folio = vm_normal_folio();

    if (!folio)

        *nr =1, return NULL;

    if the other skipping conditions happen, goto skip_batch, return folio and set *nr = batch

    if (sysctl.....) {

skip_batch:

    *nr = mprotect_batch();

}

    if (folio_use_access_time)....

    return folio;

}

#9 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 15 lines · Dev Jain
Quoted 13 lines · David Hildenbrand
Quoted 12 lines
Quoted 2 lines

... likely with a better function name,

I want to be able to reuse the folio from vm_normal_folio(), and we also need

nr_ptes to know how much to skip, so if there is no objection in passing int *nr_ptes,

or struct folio **foliop to this new function, then I'll carry on with your suggestion :)

Quoted 7 lines
Quoted 2 lines

Can you quickly prototype what you have in mind and paste it here? Will make it easier :)

if (prot_numa)

      func(vma, addr, oldpte, &nr);

I'd probably return "nr_ptes" and return the folio using a &folio instead.

That way, you can easily extend the function to return the folio in the patch where you really need it (not this patch IIUR :) )

Signature · 3 lines
Cheers,

David / dhildenb
author
#10 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 20 lines · David Hildenbrand
Quoted 19 lines · Dev Jain
Quoted 15 lines · David Hildenbrand
Quoted 4 lines
Quoted 2 lines

... likely with a better function name,

Quoted 8 lines

I want to be able to reuse the folio from vm_normal_folio(), and we also need

nr_ptes to know how much to skip, so if there is no objection in passing int *nr_ptes,

or struct folio **foliop to this new function, then I'll carry on with your suggestion :)

Can you quickly prototype what you have in mind and paste it here? Will make it easier :)

Quoted 7 lines
Quoted 3 lines

if (prot_numa)

      func(vma, addr, oldpte, &nr);

I'd probably return "nr_ptes" and return the folio using a &folio instead.

That way, you can easily extend the function to return the folio in the patch where you really need it (not this patch IIUR :) )

Just confirming, you mean to return nr_ptes and get the folio by passing &folio, and the function parameter will be struct folio **foliop?

#11 lore raw

Re: [PATCH v3 1/5] mm: Optimize mprotect() by batch-skipping PTEs

Quoted 28 lines · Dev Jain
Quoted 26 lines · David Hildenbrand
Quoted 24 lines · Dev Jain
Quoted 12 lines · David Hildenbrand
Quoted 12 lines
Quoted 2 lines

... likely with a better function name,

I want to be able to reuse the folio from vm_normal_folio(), and we also
need

nr_ptes to know how much to skip, so if there is no objection in passing
int *nr_ptes,

Quoted 5 lines
Quoted 2 lines

or struct folio **foliop to this new function, then I'll carry on with your suggestion :)

Can you quickly prototype what you have in mind and paste it here? Will make it easier :)

if (prot_numa)

      func(vma, addr, oldpte, &nr);

Quoted 8 lines
Quoted 5 lines

I'd probably return "nr_ptes" and return the folio using a &folio instead.

That way, you can easily extend the function to return the folio in the patch where you really need it (not this patch IIUR :) )

Just confirming, you mean to return nr_ptes and get the folio by passing &folio, and the function parameter will be struct folio **foliop?

Yes.

Signature · 3 lines
Cheers,

David / dhildenb
author
#12 lore raw

PATCH v3 2/5 mm: Add batched versions of ptep_modify_prot_start/commit

In reply to Dev Jain

Batch ptep_modify_prot_start/commit in preparation for optimizing mprotect. Architecture can override these helpers; in case not, they are implemented as a simple loop over the corresponding single pte helpers.

---
 include/linux/pgtable.h | 75 +++++++++++++++++++++++++++++++++++++++++
 mm/mprotect.c           |  4 +--
 2 files changed, 77 insertions(+), 2 deletions(-)
include/linux/pgtable.h +75 -0
index b50447ef1c92..e40ed57e034d 100644
@@ -1333,6 +1333,81 @@ static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
 	__ptep_modify_prot_commit(vma, addr, ptep, pte);
 }
 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
+
+/**
+ * modify_prot_start_ptes - Start a pte protection read-modify-write transaction
+ * over a batch of ptes, which protects against asynchronous hardware modifications
+ * to the ptes. The intention is not to prevent the hardware from making pte
+ * updates, but to prevent any updates it may make from being lost.
+ * Please see the comment above ptep_modify_prot_start() for full description.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_start(), collecting the a/d bits of the mapped
+ * folio.
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+#ifndef modify_prot_start_ptes
+static inline pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep, unsigned int nr)
+{
+	pte_t pte, tmp_pte;
+
+	pte = ptep_modify_prot_start(vma, addr, ptep);
+	while (--nr) {
+		ptep++;
+		addr += PAGE_SIZE;
+		tmp_pte = ptep_modify_prot_start(vma, addr, ptep);
+		if (pte_dirty(tmp_pte))
+			pte = pte_mkdirty(pte);
+		if (pte_young(tmp_pte))
+			pte = pte_mkyoung(pte);
+	}
+	return pte;
+}
+#endif
+
+/**
+ * modify_prot_commit_ptes - Commit an update to a batch of ptes, leaving any
+ * hardware-controlled bits in the PTE unmodified.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_commit().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+#ifndef modify_prot_commit_ptes
+static inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t old_pte, pte_t pte, unsigned int nr)
+{
+	int i;
+
+	for (i = 0; i < nr; ++i) {
+		ptep_modify_prot_commit(vma, addr, ptep, old_pte, pte);
+		ptep++;
+		addr += PAGE_SIZE;
+		old_pte = pte_next_pfn(old_pte);
+		pte = pte_next_pfn(pte);
+	}
+}
+#endif
+
 #endif /* CONFIG_MMU */
 
 /*
mm/mprotect.c +2 -2
index 1ee160ed0b14..124612ce3d24 100644
@@ -188,7 +188,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 						jiffies_to_msecs(jiffies));
 			}
 
-			oldpte = ptep_modify_prot_start(vma, addr, pte);
+			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
 			ptent = pte_modify(oldpte, newprot);
 
 			if (uffd_wp)
@@ -214,7 +214,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 			    can_change_pte_writable(vma, addr, ptent))
 				ptent = pte_mkwrite(ptent, vma);
 
-			ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
+			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
 			if (pte_needs_flush(oldpte, ptent))
 				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
 			pages++;
Signature · 1 line
2.30.2
#13 lore raw

Re: [PATCH v3 2/5] mm: Add batched versions of ptep_modify_prot_start/commit

Quoted 15 lines · Dev Jain

Batch ptep_modify_prot_start/commit in preparation for optimizing mprotect. Architecture can override these helpers; in case not, they are implemented as a simple loop over the corresponding single pte helpers.

---
 include/linux/pgtable.h | 75 +++++++++++++++++++++++++++++++++++++++++
 mm/mprotect.c           |  4 +--
 2 files changed, 77 insertions(+), 2 deletions(-)
include/linux/pgtable.h +0 -0
index b50447ef1c92..e40ed57e034d 100644

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1339

Quoted 7 lines
include/linux/pgtable.h +4 -0
@@ -1333,6 +1333,81 @@ static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
 	__ptep_modify_prot_commit(vma, addr, ptep, pte);
 }
 #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
+
+/**
+ * modify_prot_start_ptes - Start a pte protection read-modify-write transaction
+ * over a batch of ptes, which protects against asynchronous hardware modifications

nit: This overflows the 80 char soft limit.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1351

Quoted 12 lines
include/linux/pgtable.h +12 -0
+ * to the ptes. The intention is not to prevent the hardware from making pte
+ * updates, but to prevent any updates it may make from being lost.
+ * Please see the comment above ptep_modify_prot_start() for full description.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_start(), collecting the a/d bits of the mapped
+ * folio.

nit: "mapped folio" is a bit confusing given we are operating on ptes. Perhaps "collecting the a/d bits from each pte in the batch" is clearer.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1353

Quoted 2 lines
include/linux/pgtable.h +2 -0
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.

nit: Perhaps "batch" would be more consistent than "range"?

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1360

Quoted 7 lines
include/linux/pgtable.h +7 -0
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+#ifndef modify_prot_start_ptes
+static inline pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep, unsigned int nr)

I thought David H suggested modify_prot_ptes_start() and modify_prot_ptes_commit(), which we settled on? I'm personally fine with either though.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1365

Quoted 5 lines
include/linux/pgtable.h +5 -0
+{
+	pte_t pte, tmp_pte;
+
+	pte = ptep_modify_prot_start(vma, addr, ptep);
+	while (--nr) {

I thought we agreed to make the loop logic a bit more standard. I don't recall exactly what was finally agreed, but I would think something like this would be better:

	for (i = 1; i < nr; i++) {
Quoted 9 lines
include/linux/pgtable.h +9 -0
+		ptep++;
+		addr += PAGE_SIZE;
+		tmp_pte = ptep_modify_prot_start(vma, addr, ptep);
+		if (pte_dirty(tmp_pte))
+			pte = pte_mkdirty(pte);
+		if (pte_young(tmp_pte))
+			pte = pte_mkyoung(pte);
+	}
+	return pte;

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1384

Quoted 10 lines
include/linux/pgtable.h +10 -0
+}
+#endif
+
+/**
+ * modify_prot_commit_ptes - Commit an update to a batch of ptes, leaving any
+ * hardware-controlled bits in the PTE unmodified.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.

You've missed pte and old_pte params here.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1390

Quoted 6 lines
include/linux/pgtable.h +6 -0
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_commit().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.

How can it? All the applied bits other than the PFN will be exactly the same for the range because they all come from pte. I think this line can be dropped.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1356

Quoted 3 lines
include/linux/pgtable.h +3 -0
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.

The middle sentance doesn't apply; the PTEs will all initially be none if using the default version of modify_prot_start_ptes(). I think that can be dropped. But I think you need to explain that this will be the case on exit.

Quoted 6 lines
include/linux/pgtable.h +6 -0
+ */
+#ifndef modify_prot_commit_ptes
+static inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t old_pte, pte_t pte, unsigned int nr)
+{
+	int i;

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1409

Quoted 10 lines
include/linux/pgtable.h +10 -0
+
+	for (i = 0; i < nr; ++i) {
+		ptep_modify_prot_commit(vma, addr, ptep, old_pte, pte);
+		ptep++;
+		addr += PAGE_SIZE;
+		old_pte = pte_next_pfn(old_pte);
+		pte = pte_next_pfn(pte);
+	}
+}
+#endif

I have some general concerns about the correctness of batching these functions. The support was originally added by Commit 1ea0704e0da6 ("mm: add a ptep_modify_prot transaction abstraction"), and the intent was to make it easier to defer the pte updates for XEN on x86.

Your default implementations of the batched versions will match the number of ptep_modify_prot_start() calls with the same number of ptep_modify_prot_commit() calls, even if modify_prot_commit_ptes() is called incrementally for sub-batches of the batch used for modify_prot_start_ptes(). That's a requirement and you've met it. But in the batched case, there are 2 differences;

- You can now have multiple PTEs within a start-commit block at one time. I
hope none of the specialized implementations care about that (i.e. XEN).

- when calling ptep_modify_prot_commit(), old_pte may not be exactly what
ptep_modify_prot_start() returned for that pte. You have collected the A/D bits, and according to your docs "PTE bits in the PTE range besides the PFN can differ" when calling modify_prot_start_ptes() so R/W and other things could differ here.

I'm not sure if these are problems in practice; they probably are not. But have you checked the XEN implementation (and any other specialized implementations) are definitely compatible with your batched semantics?

Thanks,
Ryan

Quoted 26 lines
include/linux/pgtable.h +1 -0
+
 #endif /* CONFIG_MMU */
 
 /*
mm/mprotect.c +2 -2
index 1ee160ed0b14..124612ce3d24 100644
@@ -188,7 +188,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 						jiffies_to_msecs(jiffies));
 			}
 
-			oldpte = ptep_modify_prot_start(vma, addr, pte);
+			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
 			ptent = pte_modify(oldpte, newprot);
 
 			if (uffd_wp)
@@ -214,7 +214,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 			    can_change_pte_writable(vma, addr, ptent))
 				ptent = pte_mkwrite(ptent, vma);
 
-			ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
+			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
 			if (pte_needs_flush(oldpte, ptent))
 				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
 			pages++;
#14 lore raw

Re: [PATCH v3 2/5] mm: Add batched versions of ptep_modify_prot_start/commit

Quoted 6 lines · Ryan Roberts
Quoted 5 lines · Dev Jain

Batch ptep_modify_prot_start/commit in preparation for optimizing mprotect. Architecture can override these helpers; in case not, they are implemented as a simple loop over the corresponding single pte helpers.

[...]

Quoted 6 lines

I have some general concerns about the correctness of batching these functions. The support was originally added by Commit 1ea0704e0da6 ("mm: add a ptep_modify_prot transaction abstraction"), and the intent was to make it easier to defer the pte updates for XEN on x86.

Quoted 8 lines

Your default implementations of the batched versions will match the number of ptep_modify_prot_start() calls with the same number of ptep_modify_prot_commit() calls, even if modify_prot_commit_ptes() is called incrementally for sub-batches of the batch used for modify_prot_start_ptes(). That's a requirement and you've met it. But in the batched case, there are 2 differences;

- You can now have multiple PTEs within a start-commit block at one time. I
hope none of the specialized implementations care about that (i.e. XEN).

I had a look; this isn't a problem.

Quoted 6 lines

- when calling ptep_modify_prot_commit(), old_pte may not be exactly what
ptep_modify_prot_start() returned for that pte. You have collected the A/D bits, and according to your docs "PTE bits in the PTE range besides the PFN can differ" when calling modify_prot_start_ptes() so R/W and other things could differ here.

It looks like powerpc will break if you provide old_pte which has different permissions to the "real" old_pte, see radix__ptep_modify_prot_commit(). So I think you need to at least spec modify_prot_start_ptes() to require that all bits of the PTE except the PFN, access and dirty are identical. And perhaps you can VM_WARN if found to be otherwise? And perhaps modify ptep_modify_prot_commit()'s documentation to explcitly allow old_pte's access/dirty to be "upgraded" from what was actually read in ptep_modify_prot_start()?

XEN/x86 and arm64 don't care about old_pte.

Thanks,
Ryan

Quoted 5 lines

I'm not sure if these are problems in practice; they probably are not. But have you checked the XEN implementation (and any other specialized implementations) are definitely compatible with your batched semantics?

author
#15 lore raw

Re: [PATCH v3 2/5] mm: Add batched versions of ptep_modify_prot_start/commit

Quoted 29 lines · Ryan Roberts
Quoted 6 lines · Ryan Roberts
Quoted 5 lines · Dev Jain

Batch ptep_modify_prot_start/commit in preparation for optimizing mprotect. Architecture can override these helpers; in case not, they are implemented as a simple loop over the corresponding single pte helpers.

[...]

Quoted 5 lines

I have some general concerns about the correctness of batching these functions. The support was originally added by Commit 1ea0704e0da6 ("mm: add a ptep_modify_prot transaction abstraction"), and the intent was to make it easier to defer the pte updates for XEN on x86.

Quoted 8 lines

Your default implementations of the batched versions will match the number of ptep_modify_prot_start() calls with the same number of ptep_modify_prot_commit() calls, even if modify_prot_commit_ptes() is called incrementally for sub-batches of the batch used for modify_prot_start_ptes(). That's a requirement and you've met it. But in the batched case, there are 2 differences;

- You can now have multiple PTEs within a start-commit block at one time. I
hope none of the specialized implementations care about that (i.e. XEN).

I had a look; this isn't a problem.

Quoted 5 lines

- when calling ptep_modify_prot_commit(), old_pte may not be exactly what
ptep_modify_prot_start() returned for that pte. You have collected the A/D bits, and according to your docs "PTE bits in the PTE range besides the PFN can differ" when calling modify_prot_start_ptes() so R/W and other things could differ here.

Quoted 8 lines

It looks like powerpc will break if you provide old_pte which has different permissions to the "real" old_pte, see radix__ptep_modify_prot_commit(). So I think you need to at least spec modify_prot_start_ptes() to require that all bits of the PTE except the PFN, access and dirty are identical. And perhaps you can VM_WARN if found to be otherwise? And perhaps modify ptep_modify_prot_commit()'s documentation to explcitly allow old_pte's access/dirty to be "upgraded" from what was actually read in ptep_modify_prot_start()?

Got it, so we just need to document that, the permissions for all ptes must be identical

when using modify_prot_start_ptes(). And that we may be smearing extra a/d bits in

modify_prot_commit_ptes().

Quoted 10 lines

XEN/x86 and arm64 don't care about old_pte.

Thanks,
Ryan

Quoted 4 lines

I'm not sure if these are problems in practice; they probably are not. But have you checked the XEN implementation (and any other specialized implementations) are definitely compatible with your batched semantics?

#16 lore raw

Re: [PATCH v3 2/5] mm: Add batched versions of ptep_modify_prot_start/commit

Quoted 41 lines · Dev Jain
Quoted 29 lines · Ryan Roberts
Quoted 6 lines · Ryan Roberts
Quoted 5 lines · Dev Jain

Batch ptep_modify_prot_start/commit in preparation for optimizing mprotect. Architecture can override these helpers; in case not, they are implemented as a simple loop over the corresponding single pte helpers.

[...]

Quoted 5 lines

I have some general concerns about the correctness of batching these functions. The support was originally added by Commit 1ea0704e0da6 ("mm: add a ptep_modify_prot transaction abstraction"), and the intent was to make it easier to defer the pte updates for XEN on x86.

Quoted 8 lines

Your default implementations of the batched versions will match the number of ptep_modify_prot_start() calls with the same number of ptep_modify_prot_commit() calls, even if modify_prot_commit_ptes() is called incrementally for sub-batches of the batch used for modify_prot_start_ptes(). That's a requirement and you've met it. But in the batched case, there are 2 differences;

   - You can now have multiple PTEs within a start-commit block at one time. I hope none of the specialized implementations care about that (i.e. XEN).

I had a look; this isn't a problem.

Quoted 5 lines

   - when calling ptep_modify_prot_commit(), old_pte may not be exactly what ptep_modify_prot_start() returned for that pte. You have collected the A/D bits, and according to your docs "PTE bits in the PTE range besides the PFN can differ" when calling modify_prot_start_ptes() so R/W and other things could differ here.

Quoted 8 lines

It looks like powerpc will break if you provide old_pte which has different permissions to the "real" old_pte, see radix__ptep_modify_prot_commit(). So I think you need to at least spec modify_prot_start_ptes() to require that all bits of the PTE except the PFN, access and dirty are identical. And perhaps you can VM_WARN if found to be otherwise? And perhaps modify ptep_modify_prot_commit()'s documentation to explcitly allow old_pte's access/dirty to be "upgraded" from what was actually read in ptep_modify_prot_start()?

Quoted 2 lines

Got it, so we just need to document that, the permissions for all ptes must be identical

Not just permissions; all bits (inc SW bits) except PFN and A/D.

Quoted 16 lines

when using modify_prot_start_ptes(). And that we may be smearing extra a/d bits in

modify_prot_commit_ptes().

Quoted 10 lines

XEN/x86 and arm64 don't care about old_pte.

Thanks,
Ryan

Quoted 4 lines

I'm not sure if these are problems in practice; they probably are not. But have you checked the XEN implementation (and any other specialized implementations) are definitely compatible with your batched semantics?

author
#17 lore raw

Re: [PATCH v3 2/5] mm: Add batched versions of ptep_modify_prot_start/commit

In reply to Ryan Roberts
Quoted 37 lines · Ryan Roberts
Quoted 15 lines · Dev Jain

Batch ptep_modify_prot_start/commit in preparation for optimizing mprotect. Architecture can override these helpers; in case not, they are implemented as a simple loop over the corresponding single pte helpers.

---
  include/linux/pgtable.h | 75 +++++++++++++++++++++++++++++++++++++++++
  mm/mprotect.c           |  4 +--
  2 files changed, 77 insertions(+), 2 deletions(-)
include/linux/pgtable.h +0 -0
index b50447ef1c92..e40ed57e034d 100644

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1339

Quoted 7 lines
include/linux/pgtable.h +4 -0
@@ -1333,6 +1333,81 @@ static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
  	__ptep_modify_prot_commit(vma, addr, ptep, pte);
  }
  #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
+
+/**
+ * modify_prot_start_ptes - Start a pte protection read-modify-write transaction
+ * over a batch of ptes, which protects against asynchronous hardware modifications

nit: This overflows the 80 char soft limit.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1351

Quoted 12 lines
include/linux/pgtable.h +12 -0
+ * to the ptes. The intention is not to prevent the hardware from making pte
+ * updates, but to prevent any updates it may make from being lost.
+ * Please see the comment above ptep_modify_prot_start() for full description.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_start(), collecting the a/d bits of the mapped
+ * folio.
Quoted 2 lines

nit: "mapped folio" is a bit confusing given we are operating on ptes. Perhaps "collecting the a/d bits from each pte in the batch" is clearer.

Sure.

Quoted 4 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1353

Quoted 3 lines
include/linux/pgtable.h +2 -0
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.

nit: Perhaps "batch" would be more consistent than "range"?

Sure.

Quoted 11 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1360

Quoted 8 lines
include/linux/pgtable.h +7 -0
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+#ifndef modify_prot_start_ptes
+static inline pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep, unsigned int nr)

I thought David H suggested modify_prot_ptes_start() and modify_prot_ptes_commit(), which we settled on? I'm personally fine with either though.

No strong opinion, I'll do that.

Quoted 11 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1365

Quoted 6 lines
include/linux/pgtable.h +5 -0
+{
+	pte_t pte, tmp_pte;
+
+	pte = ptep_modify_prot_start(vma, addr, ptep);
+	while (--nr) {

I thought we agreed to make the loop logic a bit more standard. I don't recall exactly what was finally agreed, but I would think something like this would be better:

	for (i = 1; i < nr; i++) {

Sure.

Quoted 13 lines
Quoted 13 lines
include/linux/pgtable.h +12 -0
+		ptep++;
+		addr += PAGE_SIZE;
+		tmp_pte = ptep_modify_prot_start(vma, addr, ptep);
+		if (pte_dirty(tmp_pte))
+			pte = pte_mkdirty(pte);
+		if (pte_young(tmp_pte))
+			pte = pte_mkyoung(pte);
+	}
+	return pte;
+}
+#endif
+
Quoted 8 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1384

Quoted 7 lines
include/linux/pgtable.h +7 -0
+/**
+ * modify_prot_commit_ptes - Commit an update to a batch of ptes, leaving any
+ * hardware-controlled bits in the PTE unmodified.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.

You've missed pte and old_pte params here.

My bad.

Quoted 9 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1390

Quoted 7 lines
include/linux/pgtable.h +6 -0
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_commit().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.

How can it? All the applied bits other than the PFN will be exactly the same for the range because they all come from pte. I think this line can be dropped.

Copy pasted, then forgot to remove :)

Quoted 7 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1356

Quoted 4 lines
include/linux/pgtable.h +3 -0
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.

The middle sentance doesn't apply; the PTEs will all initially be none if using the default version of modify_prot_start_ptes(). I think that can be dropped. But I think you need to explain that this will be the case on exit.

Ah got it. "On exit, the set ptes will map the same folio."

Quoted 70 lines
Quoted 7 lines
include/linux/pgtable.h +6 -0
+ */
+#ifndef modify_prot_commit_ptes
+static inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t old_pte, pte_t pte, unsigned int nr)
+{
+	int i;

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1409

Quoted 10 lines
include/linux/pgtable.h +10 -0
+
+	for (i = 0; i < nr; ++i) {
+		ptep_modify_prot_commit(vma, addr, ptep, old_pte, pte);
+		ptep++;
+		addr += PAGE_SIZE;
+		old_pte = pte_next_pfn(old_pte);
+		pte = pte_next_pfn(pte);
+	}
+}
+#endif

I have some general concerns about the correctness of batching these functions. The support was originally added by Commit 1ea0704e0da6 ("mm: add a ptep_modify_prot transaction abstraction"), and the intent was to make it easier to defer the pte updates for XEN on x86.

Your default implementations of the batched versions will match the number of ptep_modify_prot_start() calls with the same number of ptep_modify_prot_commit() calls, even if modify_prot_commit_ptes() is called incrementally for sub-batches of the batch used for modify_prot_start_ptes(). That's a requirement and you've met it. But in the batched case, there are 2 differences;

- You can now have multiple PTEs within a start-commit block at one time. I
hope none of the specialized implementations care about that (i.e. XEN).

- when calling ptep_modify_prot_commit(), old_pte may not be exactly what
ptep_modify_prot_start() returned for that pte. You have collected the A/D bits, and according to your docs "PTE bits in the PTE range besides the PFN can differ" when calling modify_prot_start_ptes() so R/W and other things could differ here.

I'm not sure if these are problems in practice; they probably are not. But have you checked the XEN implementation (and any other specialized implementations) are definitely compatible with your batched semantics?

Thanks,
Ryan

Quoted 26 lines
include/linux/pgtable.h +1 -0
+
  #endif /* CONFIG_MMU */
 
  /*
mm/mprotect.c +2 -2
index 1ee160ed0b14..124612ce3d24 100644
@@ -188,7 +188,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  						jiffies_to_msecs(jiffies));
  			}
 
-			oldpte = ptep_modify_prot_start(vma, addr, pte);
+			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
  			ptent = pte_modify(oldpte, newprot);
 
  			if (uffd_wp)
@@ -214,7 +214,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  			    can_change_pte_writable(vma, addr, ptent))
  				ptent = pte_mkwrite(ptent, vma);
 
-			ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
+			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
  			if (pte_needs_flush(oldpte, ptent))
  				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
  			pages++;
author
#18 lore raw

Re: [PATCH v3 2/5] mm: Add batched versions of ptep_modify_prot_start/commit

In reply to Ryan Roberts
Quoted 44 lines · Ryan Roberts
Quoted 15 lines · Dev Jain

Batch ptep_modify_prot_start/commit in preparation for optimizing mprotect. Architecture can override these helpers; in case not, they are implemented as a simple loop over the corresponding single pte helpers.

---
  include/linux/pgtable.h | 75 +++++++++++++++++++++++++++++++++++++++++
  mm/mprotect.c           |  4 +--
  2 files changed, 77 insertions(+), 2 deletions(-)
include/linux/pgtable.h +0 -0
index b50447ef1c92..e40ed57e034d 100644

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1339

Quoted 7 lines
include/linux/pgtable.h +4 -0
@@ -1333,6 +1333,81 @@ static inline void ptep_modify_prot_commit(struct vm_area_struct *vma,
  	__ptep_modify_prot_commit(vma, addr, ptep, pte);
  }
  #endif /* __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION */
+
+/**
+ * modify_prot_start_ptes - Start a pte protection read-modify-write transaction
+ * over a batch of ptes, which protects against asynchronous hardware modifications

nit: This overflows the 80 char soft limit.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1351

Quoted 12 lines
include/linux/pgtable.h +12 -0
+ * to the ptes. The intention is not to prevent the hardware from making pte
+ * updates, but to prevent any updates it may make from being lost.
+ * Please see the comment above ptep_modify_prot_start() for full description.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_start(), collecting the a/d bits of the mapped
+ * folio.

nit: "mapped folio" is a bit confusing given we are operating on ptes. Perhaps "collecting the a/d bits from each pte in the batch" is clearer.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1353

Quoted 2 lines
include/linux/pgtable.h +2 -0
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.

nit: Perhaps "batch" would be more consistent than "range"?

Quoted 10 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1360

Quoted 7 lines
include/linux/pgtable.h +7 -0
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.
+ */
+#ifndef modify_prot_start_ptes
+static inline pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
+		unsigned long addr, pte_t *ptep, unsigned int nr)

I thought David H suggested modify_prot_ptes_start() and modify_prot_ptes_commit(), which we settled on? I'm personally fine with either though.

Nothing was decided upon strongly. I will personally keep it the same for sake of consistency.

Quoted 8 lines

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1365

Quoted 5 lines
include/linux/pgtable.h +5 -0
+{
+	pte_t pte, tmp_pte;
+
+	pte = ptep_modify_prot_start(vma, addr, ptep);
+	while (--nr) {

I thought we agreed to make the loop logic a bit more standard. I don't recall exactly what was finally agreed, but I would think something like this would be better:

Again, nothing was particularly decided on as far as I remember : ) Let us
keep it the same - https://lore.kernel.org/all/[email protected]/

Quoted 108 lines
	for (i = 1; i < nr; i++) {
Quoted 9 lines
include/linux/pgtable.h +9 -0
+		ptep++;
+		addr += PAGE_SIZE;
+		tmp_pte = ptep_modify_prot_start(vma, addr, ptep);
+		if (pte_dirty(tmp_pte))
+			pte = pte_mkdirty(pte);
+		if (pte_young(tmp_pte))
+			pte = pte_mkyoung(pte);
+	}
+	return pte;

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1384

Quoted 10 lines
include/linux/pgtable.h +10 -0
+}
+#endif
+
+/**
+ * modify_prot_commit_ptes - Commit an update to a batch of ptes, leaving any
+ * hardware-controlled bits in the PTE unmodified.
+ *
+ * @vma: The virtual memory area the pages are mapped into.
+ * @addr: Address the first page is mapped at.
+ * @ptep: Page table pointer for the first entry.

You've missed pte and old_pte params here.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1390

Quoted 6 lines
include/linux/pgtable.h +6 -0
+ * @nr: Number of entries.
+ *
+ * May be overridden by the architecture; otherwise, implemented as a simple
+ * loop over ptep_modify_prot_commit().
+ *
+ * Note that PTE bits in the PTE range besides the PFN can differ.

How can it? All the applied bits other than the PFN will be exactly the same for the range because they all come from pte. I think this line can be dropped.

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1356

Quoted 3 lines
include/linux/pgtable.h +3 -0
+ *
+ * Context: The caller holds the page table lock.  The PTEs map consecutive
+ * pages that belong to the same folio.  The PTEs are all in the same PMD.

The middle sentance doesn't apply; the PTEs will all initially be none if using the default version of modify_prot_start_ptes(). I think that can be dropped. But I think you need to explain that this will be the case on exit.

Quoted 6 lines
include/linux/pgtable.h +6 -0
+ */
+#ifndef modify_prot_commit_ptes
+static inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+		pte_t *ptep, pte_t old_pte, pte_t pte, unsigned int nr)
+{
+	int i;

include/linux/pgtable.h static inline void ptep_modify_prot_commit(struct vm_area_struct *vma, · line 1409

Quoted 10 lines
include/linux/pgtable.h +10 -0
+
+	for (i = 0; i < nr; ++i) {
+		ptep_modify_prot_commit(vma, addr, ptep, old_pte, pte);
+		ptep++;
+		addr += PAGE_SIZE;
+		old_pte = pte_next_pfn(old_pte);
+		pte = pte_next_pfn(pte);
+	}
+}
+#endif

I have some general concerns about the correctness of batching these functions. The support was originally added by Commit 1ea0704e0da6 ("mm: add a ptep_modify_prot transaction abstraction"), and the intent was to make it easier to defer the pte updates for XEN on x86.

Your default implementations of the batched versions will match the number of ptep_modify_prot_start() calls with the same number of ptep_modify_prot_commit() calls, even if modify_prot_commit_ptes() is called incrementally for sub-batches of the batch used for modify_prot_start_ptes(). That's a requirement and you've met it. But in the batched case, there are 2 differences;

- You can now have multiple PTEs within a start-commit block at one time. I
hope none of the specialized implementations care about that (i.e. XEN).

- when calling ptep_modify_prot_commit(), old_pte may not be exactly what
ptep_modify_prot_start() returned for that pte. You have collected the A/D bits, and according to your docs "PTE bits in the PTE range besides the PFN can differ" when calling modify_prot_start_ptes() so R/W and other things could differ here.

I'm not sure if these are problems in practice; they probably are not. But have you checked the XEN implementation (and any other specialized implementations) are definitely compatible with your batched semantics?

Thanks,
Ryan

Quoted 26 lines
include/linux/pgtable.h +1 -0
+
  #endif /* CONFIG_MMU */
 
  /*
mm/mprotect.c +2 -2
index 1ee160ed0b14..124612ce3d24 100644
@@ -188,7 +188,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  						jiffies_to_msecs(jiffies));
  			}
 
-			oldpte = ptep_modify_prot_start(vma, addr, pte);
+			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
  			ptent = pte_modify(oldpte, newprot);
 
  			if (uffd_wp)
@@ -214,7 +214,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  			    can_change_pte_writable(vma, addr, ptent))
  				ptent = pte_mkwrite(ptent, vma);
 
-			ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
+			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
  			if (pte_needs_flush(oldpte, ptent))
  				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
  			pages++;
author
#19 lore raw

PATCH v3 3/5 mm: Optimize mprotect() by PTE batching

In reply to Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).
For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is), the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same PageAnonExclusive condition, and process that sub batch, then determine and process the next sub batch, and so on. Note that this does not cause any extra overhead; if suppose the size of the folio batch is 512, then the sub batch processing in total will take 512 iterations, which is the same as what we would have done before.

---
 include/linux/mm.h |   7 ++-
 mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
 2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +5 -2
index 43748c8f3454..7d5b96f005dc 100644
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
 #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
 					    MM_CP_UFFD_WP_RESOLVE)
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)	\
+	can_change_ptes_writable(vma, addr, pte, 1, NULL)
+
 extern long change_protection(struct mmu_gather *tlb,
 			      struct vm_area_struct *vma, unsigned long start,
 			      unsigned long end, unsigned long cp_flags);
mm/mprotect.c +99 -27

204 lines not on this page · raw message

Signature · 1 line
2.30.2
#20 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

Quoted 100 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).
For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is), the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same PageAnonExclusive condition, and process that sub batch, then determine and process the next sub batch, and so on. Note that this does not cause any extra overhead; if suppose the size of the folio batch is 512, then the sub batch processing in total will take 512 iterations, which is the same as what we would have done before.

---
 include/linux/mm.h |   7 ++-
 mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
 2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +5 -2
index 43748c8f3454..7d5b96f005dc 100644
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
 #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
                                            MM_CP_UFFD_WP_RESOLVE)
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                            pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                            pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)        \
+       can_change_ptes_writable(vma, addr, pte, 1, NULL)
+
 extern long change_protection(struct mmu_gather *tlb,
                              struct vm_area_struct *vma, unsigned long start,
                              unsigned long end, unsigned long cp_flags);
mm/mprotect.c +21 -10
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
 
 #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                            pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                            pte_t pte, int max_len, int *len)
 {
        struct page *page;
+       bool temp_ret;
+       bool ret;
+       int i;
 
-       if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-               return false;
+       if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+               ret = false;
+               goto out;
+       }
 
        /* Don't touch entries that are not even readable. */
-       if (pte_protnone(pte))
-               return false;
+       if (pte_protnone(pte)) {
+               ret = false;
+               goto out;
+       }
 
        /* Do we need write faults for softdirty tracking? */
-       if (pte_needs_soft_dirty_wp(vma, pte))
-               return false;
+       if (pte_needs_soft_dirty_wp(vma, pte)) {
+               ret = false;
+               goto out;
+       }
 
        /* Do we need write faults for uffd-wp tracking? */
-       if (userfaultfd_pte_wp(vma, pte))
-               return false;
+       if (userfaultfd_pte_wp(vma, pte)) {
+               ret = false;
+               goto out;
+       }
 
        if (!(vma->vm_flags & VM_SHARED)) {
                /*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
                 * any additional checks while holding the PT lock.
                 */
                page = vm_normal_page(vma, addr, pte);

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 90

Quoted 10 lines
mm/mprotect.c +9 -1
-               return page && PageAnon(page) && PageAnonExclusive(page);
+               ret = (page && PageAnon(page) && PageAnonExclusive(page));
+               if (!len)
+                       return ret;
+
+               /* Check how many consecutive pages are AnonExclusive or not */
+               for (i = 1; i < max_len; ++i) {
+                       ++page;
+                       temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+                       if (temp_ret != ret)

Do we really need to do PageAnon for each subpage which is:
folio_test_anon(page_folio(page))

since we have checked subpage[0] ?

Quoted 153 lines
mm/mprotect.c +69 -17
+                               break;
+               }
+               *len = i;
+               return ret;
        }
 
        VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
         * FS was already notified and we can simply mark the PTE writable
         * just like the write-fault handler would do.
         */
-       return pte_dirty(pte);
+       ret = pte_dirty(pte);
+
+out:
+       /* The entire batch is guaranteed to have the same return value */
+       if (len)
+               *len = max_len;
+       return ret;
 }
 
 static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-               pte_t pte, int max_nr_ptes)
+               pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)
 {
-       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       fpb_t flags = FPB_IGNORE_DIRTY;
 
-       if (!folio_test_large(folio) || (max_nr_ptes == 1))
+       if (ignore_soft_dirty)
+               flags |= FPB_IGNORE_SOFT_DIRTY;
+
+       if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
                return 1;
 
        return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
                               NULL, NULL, NULL);
 }
 
+/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+               unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+               int max_len, int sub_batch_idx)
+{
+       unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+       pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+       pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+       pte_t *new_ptep = ptep + sub_batch_idx;
+       int len = 1;
+
+       if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+               new_ptent = pte_mkwrite(new_ptent, vma);
+
+       modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent, len);
+       if (pte_needs_flush(new_oldpte, new_ptent))
+               tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE);
+       return len;
+}
+
 static long change_pte_range(struct mmu_gather *tlb,
                struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
                unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
        bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
        bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
        bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-       int nr_ptes;
+       int sub_batch_idx, max_len, len, nr_ptes;
 
        tlb_change_page_size(tlb, PAGE_SIZE);
        pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
        flush_tlb_batched_pending(vma->vm_mm);
        arch_enter_lazy_mmu_mode();
        do {
+               sub_batch_idx = 0;
                nr_ptes = 1;
                oldpte = ptep_get(pte);
                if (pte_present(oldpte)) {
                        int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+                       struct folio *folio = NULL;
                        pte_t ptent;
 
                        /*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
                         * pages. See similar comment in change_huge_pmd.
                         */
                        if (prot_numa) {
-                               struct folio *folio;
                                int nid;
                                bool toptier;
 
@@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,
                                    toptier) {
 skip_batch:
                                        nr_ptes = mprotect_batch(folio, addr, pte,
-                                                                oldpte, max_nr_ptes);
+                                                                oldpte, max_nr_ptes,
+                                                                true);
                                        continue;
                                }
                                if (folio_use_access_time(folio))
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
                                                jiffies_to_msecs(jiffies));
                        }
 
+                       if (!folio)
+                               folio = vm_normal_folio(vma, addr, oldpte);
+
+                       nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+                                                max_nr_ptes, false);
                        oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
                        ptent = pte_modify(oldpte, newprot);
 
@@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
                         * example, if a PTE is already dirty and no other
                         * COW or special handling is required.
                         */
-                       if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-                           !pte_write(ptent) &&
-                           can_change_pte_writable(vma, addr, ptent))
-                               ptent = pte_mkwrite(ptent, vma);
-
-                       modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-                       if (pte_needs_flush(oldpte, ptent))
-                               tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-                       pages++;
+                       if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+                               max_len = nr_ptes;
+                               while (sub_batch_idx < nr_ptes) {
+
+                                       /* Get length of sub batch */
+                                       len = modify_sub_batch(vma, tlb, addr, pte,
+                                                              oldpte, ptent, max_len,
+                                                              sub_batch_idx);
+                                       sub_batch_idx += len;
+                                       max_len -= len;
+                               }
+                       } else {
+                               modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
+                               if (pte_needs_flush(oldpte, ptent))
+                                       tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
+                       }
+                       pages += nr_ptes;
                } else if (is_swap_pte(oldpte)) {
                        swp_entry_t entry = pte_to_swp_entry(oldpte);
                        pte_t newpte;
--
Quoted 2 lines

2.30.2

Thanks
barry

author
#21 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

Hi, sorry for the late reply, my Thunderbird is not working and I am not
being able to receive emails. I am manually downloading the mbox and then replying, so my email response may be slow :(

Quoted 115 lines · Barry Song
Quoted 100 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).
For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is), the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same PageAnonExclusive condition, and process that sub batch, then determine and process the next sub batch, and so on. Note that this does not cause any extra overhead; if suppose the size of the folio batch is 512, then the sub batch processing in total will take 512 iterations, which is the same as what we would have done before.

---
  include/linux/mm.h |   7 ++-
  mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
  2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +5 -2
index 43748c8f3454..7d5b96f005dc 100644
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
  #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
                                             MM_CP_UFFD_WP_RESOLVE)
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                            pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                            pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)        \
+       can_change_ptes_writable(vma, addr, pte, 1, NULL)
+
  extern long change_protection(struct mmu_gather *tlb,
                               struct vm_area_struct *vma, unsigned long start,
                               unsigned long end, unsigned long cp_flags);
mm/mprotect.c +21 -10
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
 
  #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                            pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                            pte_t pte, int max_len, int *len)
  {
         struct page *page;
+       bool temp_ret;
+       bool ret;
+       int i;
 
-       if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-               return false;
+       if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+               ret = false;
+               goto out;
+       }
 
         /* Don't touch entries that are not even readable. */
-       if (pte_protnone(pte))
-               return false;
+       if (pte_protnone(pte)) {
+               ret = false;
+               goto out;
+       }
 
         /* Do we need write faults for softdirty tracking? */
-       if (pte_needs_soft_dirty_wp(vma, pte))
-               return false;
+       if (pte_needs_soft_dirty_wp(vma, pte)) {
+               ret = false;
+               goto out;
+       }
 
         /* Do we need write faults for uffd-wp tracking? */
-       if (userfaultfd_pte_wp(vma, pte))
-               return false;
+       if (userfaultfd_pte_wp(vma, pte)) {
+               ret = false;
+               goto out;
+       }
 
         if (!(vma->vm_flags & VM_SHARED)) {
                 /*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
                  * any additional checks while holding the PT lock.
                  */
                 page = vm_normal_page(vma, addr, pte);

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 90

Quoted 10 lines
mm/mprotect.c +9 -1
-               return page && PageAnon(page) && PageAnonExclusive(page);
+               ret = (page && PageAnon(page) && PageAnonExclusive(page));
+               if (!len)
+                       return ret;
+
+               /* Check how many consecutive pages are AnonExclusive or not */
+               for (i = 1; i < max_len; ++i) {
+                       ++page;
+                       temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+                       if (temp_ret != ret)

Do we really need to do PageAnon for each subpage which is:
folio_test_anon(page_folio(page))

Quoted 1 line

since we have checked subpage[0] ?

You are correct. I just wrote it like that for consistency, I'll change it in v4.

Quoted 159 lines
Quoted 154 lines
mm/mprotect.c +69 -17
+                               break;
+               }
+               *len = i;
+               return ret;
         }
 
         VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
          * FS was already notified and we can simply mark the PTE writable
          * just like the write-fault handler would do.
          */
-       return pte_dirty(pte);
+       ret = pte_dirty(pte);
+
+out:
+       /* The entire batch is guaranteed to have the same return value */
+       if (len)
+               *len = max_len;
+       return ret;
  }
 
  static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-               pte_t pte, int max_nr_ptes)
+               pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)
  {
-       const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       fpb_t flags = FPB_IGNORE_DIRTY;
 
-       if (!folio_test_large(folio) || (max_nr_ptes == 1))
+       if (ignore_soft_dirty)
+               flags |= FPB_IGNORE_SOFT_DIRTY;
+
+       if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
                 return 1;
 
         return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
                                NULL, NULL, NULL);
  }
 
+/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+               unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+               int max_len, int sub_batch_idx)
+{
+       unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+       pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+       pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+       pte_t *new_ptep = ptep + sub_batch_idx;
+       int len = 1;
+
+       if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+               new_ptent = pte_mkwrite(new_ptent, vma);
+
+       modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent, len);
+       if (pte_needs_flush(new_oldpte, new_ptent))
+               tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE);
+       return len;
+}
+
  static long change_pte_range(struct mmu_gather *tlb,
                 struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
                 unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
         bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
         bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
         bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-       int nr_ptes;
+       int sub_batch_idx, max_len, len, nr_ptes;
 
         tlb_change_page_size(tlb, PAGE_SIZE);
         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
         flush_tlb_batched_pending(vma->vm_mm);
         arch_enter_lazy_mmu_mode();
         do {
+               sub_batch_idx = 0;
                 nr_ptes = 1;
                 oldpte = ptep_get(pte);
                 if (pte_present(oldpte)) {
                         int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+                       struct folio *folio = NULL;
                         pte_t ptent;
 
                         /*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
                          * pages. See similar comment in change_huge_pmd.
                          */
                         if (prot_numa) {
-                               struct folio *folio;
                                 int nid;
                                 bool toptier;
 
@@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,
                                     toptier) {
  skip_batch:
                                         nr_ptes = mprotect_batch(folio, addr, pte,
-                                                                oldpte, max_nr_ptes);
+                                                                oldpte, max_nr_ptes,
+                                                                true);
                                         continue;
                                 }
                                 if (folio_use_access_time(folio))
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
                                                 jiffies_to_msecs(jiffies));
                         }
 
+                       if (!folio)
+                               folio = vm_normal_folio(vma, addr, oldpte);
+
+                       nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+                                                max_nr_ptes, false);
                         oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
                         ptent = pte_modify(oldpte, newprot);
 
@@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
                          * example, if a PTE is already dirty and no other
                          * COW or special handling is required.
                          */
-                       if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-                           !pte_write(ptent) &&
-                           can_change_pte_writable(vma, addr, ptent))
-                               ptent = pte_mkwrite(ptent, vma);
-
-                       modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-                       if (pte_needs_flush(oldpte, ptent))
-                               tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-                       pages++;
+                       if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+                               max_len = nr_ptes;
+                               while (sub_batch_idx < nr_ptes) {
+
+                                       /* Get length of sub batch */
+                                       len = modify_sub_batch(vma, tlb, addr, pte,
+                                                              oldpte, ptent, max_len,
+                                                              sub_batch_idx);
+                                       sub_batch_idx += len;
+                                       max_len -= len;
+                               }
+                       } else {
+                               modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
+                               if (pte_needs_flush(oldpte, ptent))
+                                       tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
+                       }
+                       pages += nr_ptes;
                 } else if (is_swap_pte(oldpte)) {
                         swp_entry_t entry = pte_to_swp_entry(oldpte);
                         pte_t newpte;
--
Quoted 2 lines

2.30.2

Thanks
barry

#22 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

In reply to Dev Jain
Quoted 4 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).

parisc's pte_needs_flush() is a different (static) function, just coincidentally named - it takes different arguments.

But powerpc and x86 both specialize it and they consider the dirty bit. Both implementations look to me like they will function correctly but it is a bit ugly. They both conclude that if there is no change or if dirty has gone from clear to set, then no flush is needed. (flush only needed when dirty goes from set to clear). Given after your change, oldpte may have dirty set when it wasn't really set for the pte in question that means the function could detect no change when really its a clear -> set change; it gives the same answer in both cases so it's safe. Perhaps a bit fragile though...

Quoted 7 lines

For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is),

Quoted 8 lines

the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same PageAnonExclusive condition, and process that sub batch, then determine and process the next sub batch, and so on. Note that this does not cause any extra overhead; if suppose the size of the folio batch is 512, then the sub batch processing in total will take 512 iterations, which is the same as what we would have done before.

This commit log could do with some reformatting; blank lines between paragraphs and break at whatever the usual git commit log char limit is (72 chars?).

Quoted 12 lines
---
 include/linux/mm.h |   7 ++-
 mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
 2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +0 -0
index 43748c8f3454..7d5b96f005dc 100644

include/linux/mm.h int get_cmdline(struct task_struct *task, char *buffer, int buflen); · line 2548

Quoted 9 lines
include/linux/mm.h +4 -2
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
 #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
 					    MM_CP_UFFD_WP_RESOLVE)
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)	\
+	can_change_ptes_writable(vma, addr, pte, 1, NULL)

nit: add an extra tab for readability:

#define can_change_pte_writable(vma, addr, pte) \
can_change_ptes_writable(vma, addr, pte, 1, NULL)

Quoted 83 lines
include/linux/mm.h +1 -0
+
 extern long change_protection(struct mmu_gather *tlb,
 			      struct vm_area_struct *vma, unsigned long start,
 			      unsigned long end, unsigned long cp_flags);
mm/mprotect.c +36 -12
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
 
 #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len)
 {
 	struct page *page;
+	bool temp_ret;
+	bool ret;
+	int i;
 
-	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-		return false;
+	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+		ret = false;
+		goto out;
+	}
 
 	/* Don't touch entries that are not even readable. */
-	if (pte_protnone(pte))
-		return false;
+	if (pte_protnone(pte)) {
+		ret = false;
+		goto out;
+	}
 
 	/* Do we need write faults for softdirty tracking? */
-	if (pte_needs_soft_dirty_wp(vma, pte))
-		return false;
+	if (pte_needs_soft_dirty_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
 	/* Do we need write faults for uffd-wp tracking? */
-	if (userfaultfd_pte_wp(vma, pte))
-		return false;
+	if (userfaultfd_pte_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
 	if (!(vma->vm_flags & VM_SHARED)) {
 		/*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 		 * any additional checks while holding the PT lock.
 		 */
 		page = vm_normal_page(vma, addr, pte);
-		return page && PageAnon(page) && PageAnonExclusive(page);
+		ret = (page && PageAnon(page) && PageAnonExclusive(page));
+		if (!len)
+			return ret;
+
+		/* Check how many consecutive pages are AnonExclusive or not */
+		for (i = 1; i < max_len; ++i) {
+			++page;
+			temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+			if (temp_ret != ret)
+				break;
+		}
+		*len = i;
+		return ret;
 	}
 
 	VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
 	 * FS was already notified and we can simply mark the PTE writable
 	 * just like the write-fault handler would do.
 	 */
-	return pte_dirty(pte);
+	ret = pte_dirty(pte);
+

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 116

Quoted 10 lines
mm/mprotect.c +6 -1
+out:
+	/* The entire batch is guaranteed to have the same return value */
+	if (len)
+		*len = max_len;
+	return ret;
 }
 
 static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-		pte_t pte, int max_nr_ptes)
+		pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)

We'll almost certainly want more flags here in future. I wonder if it's cleaner just to pass "fpb_t extra_flags" here instead of the bool, then OR them in. You can pass 0 or FPB_IGNORE_SOFT_DIRTY at your 2 callsites. No strong opinion though.

Quoted 96 lines
mm/mprotect.c +40 -5
 {
-	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	fpb_t flags = FPB_IGNORE_DIRTY;
 
-	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+	if (ignore_soft_dirty)
+		flags |= FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
 		return 1;
 
 	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
 			       NULL, NULL, NULL);
 }
 
+/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+		unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+		int max_len, int sub_batch_idx)
+{
+	unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+	pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+	pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+	pte_t *new_ptep = ptep + sub_batch_idx;
+	int len = 1;
+
+	if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+		new_ptent = pte_mkwrite(new_ptent, vma);
+
+	modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent, len);
+	if (pte_needs_flush(new_oldpte, new_ptent))
+		tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE);
+	return len;
+}
+
 static long change_pte_range(struct mmu_gather *tlb,
 		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
 		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
 	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-	int nr_ptes;
+	int sub_batch_idx, max_len, len, nr_ptes;
 
 	tlb_change_page_size(tlb, PAGE_SIZE);
 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
 	flush_tlb_batched_pending(vma->vm_mm);
 	arch_enter_lazy_mmu_mode();
 	do {
+		sub_batch_idx = 0;
 		nr_ptes = 1;
 		oldpte = ptep_get(pte);
 		if (pte_present(oldpte)) {
 			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+			struct folio *folio = NULL;
 			pte_t ptent;
 
 			/*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
 			 * pages. See similar comment in change_huge_pmd.
 			 */
 			if (prot_numa) {
-				struct folio *folio;
 				int nid;
 				bool toptier;
 
@@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,
 				    toptier) {
 skip_batch:
 					nr_ptes = mprotect_batch(folio, addr, pte,
-								 oldpte, max_nr_ptes);
+								 oldpte, max_nr_ptes,
+								 true);
 					continue;
 				}
 				if (folio_use_access_time(folio))
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
 						jiffies_to_msecs(jiffies));
 			}
 
+			if (!folio)
+				folio = vm_normal_folio(vma, addr, oldpte);
+
+			nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+						 max_nr_ptes, false);
 			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
 			ptent = pte_modify(oldpte, newprot);
 

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 213

Quoted 5 lines
mm/mprotect.c +0 -2
@@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
 			 * example, if a PTE is already dirty and no other
 			 * COW or special handling is required.
 			 */
-			if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-			    !pte_write(ptent) &&

Don't you need to keep the !pte_write(ptent) condition here? No need to sub- batch if write is already set.

Quoted 13 lines
mm/mprotect.c +6 -7
-			    can_change_pte_writable(vma, addr, ptent))
-				ptent = pte_mkwrite(ptent, vma);
-
-			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-			if (pte_needs_flush(oldpte, ptent))
-				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-			pages++;
+			if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+				max_len = nr_ptes;
+				while (sub_batch_idx < nr_ptes) {
+
+					/* Get length of sub batch */
+					len = modify_sub_batch(vma, tlb, addr, pte,

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 291

Quoted 10 lines
mm/mprotect.c +10 -0
+							       oldpte, ptent, max_len,
+							       sub_batch_idx);
+					sub_batch_idx += len;
+					max_len -= len;
+				}
+			} else {
+				modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
+				if (pte_needs_flush(oldpte, ptent))
+					tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
+			}

This if/else block and modify_sub_block() is all pretty ugly. I wonder if something like this would be neater:

			use_sub_batch = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && 
					!pte_write(ptent) && 
					LIKE_can_change_pte_writable(vma, addr, ptent, folio);

			while (nr_ptes) {
				if (use_sub_batch)
					sub_nr_ptes = sub_batch(...);
				else
					sub_nr_ptes = nr_ptes;

				modify_prot_commit_ptes(vma, addr, pte, oldpte, 
							ptent, sub_nr_ptes);
				if (pte_needs_flush(oldpte, ptent))
					tlb_flush_pte_range(tlb, addr,
						sub_nr_ptes * PAGE_SIZE);

				addr += sub_nr_ptes * PAGE_SIZE;
				pte += sub_nr_ptes;
				oldpte = pte_advance_pfn(oldpte, sub_nr_ptes);
				ptent = pte_advance_pfn(ptent, sub_nr_ptes);
				nr_ptes -= sub_nr_ptes;
				pages += sub_nr_ptes;
			}

Where:

- LIKE_can_change_pte_writable() is similar to can_change_pte_writable() but does everything except checking the per-page exclusive flag. Note that we already have the folio so can pass that rather than calling vm_normal_page() again.

- sub_batch() can be passed the folio and the index of the first page within the folio and the max number of pages to check (nr_ptes). Then returns the number of pages where exclusive flag is the same.

Obviously they both need better names...

Thanks,
Ryan

Quoted 4 lines
mm/mprotect.c +1 -0
+			pages += nr_ptes;
 		} else if (is_swap_pte(oldpte)) {
 			swp_entry_t entry = pte_to_swp_entry(oldpte);
 			pte_t newpte;
author
#23 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

Quoted 7 lines · Ryan Roberts
Quoted 4 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).

parisc's pte_needs_flush() is a different (static) function, just coincidentally named - it takes different arguments.

Oh my god I'm *that* stupid :(

Quoted 9 lines

But powerpc and x86 both specialize it and they consider the dirty bit. Both implementations look to me like they will function correctly but it is a bit ugly. They both conclude that if there is no change or if dirty has gone from clear to set, then no flush is needed. (flush only needed when dirty goes from set to clear). Given after your change, oldpte may have dirty set when it wasn't really set for the pte in question that means the function could detect no change when really its a clear -> set change; it gives the same answer in both cases so it's safe. Perhaps a bit fragile though...

Yeah I was wondering about the fragility too. There is a hard dependency between

arch private functions doing a very specific thing so that our batching does not break.

Best we can do is add a comment explaining this, I think.

Quoted 16 lines
Quoted 16 lines

For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is), the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same PageAnonExclusive condition, and process that sub batch, then determine and process the next sub batch, and so on. Note that this does not cause any extra overhead; if suppose the size of the folio batch is 512, then the sub batch processing in total will take 512 iterations, which is the same as what we would have done before.

Quoted 2 lines

This commit log could do with some reformatting; blank lines between paragraphs and break at whatever the usual git commit log char limit is (72 chars?).

Sure, and I think it is 75 chars.

Quoted 23 lines
Quoted 12 lines
---
  include/linux/mm.h |   7 ++-
  mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
  2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +0 -0
index 43748c8f3454..7d5b96f005dc 100644

include/linux/mm.h int get_cmdline(struct task_struct *task, char *buffer, int buflen); · line 2548

Quoted 9 lines
include/linux/mm.h +4 -2
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
  #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
  					    MM_CP_UFFD_WP_RESOLVE)
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)	\
+	can_change_ptes_writable(vma, addr, pte, 1, NULL)

nit: add an extra tab for readability:

Quoted 2 lines

#define can_change_pte_writable(vma, addr, pte) \
can_change_ptes_writable(vma, addr, pte, 1, NULL)

Sure.

Quoted 91 lines
Quoted 91 lines
include/linux/mm.h +1 -0
+
  extern long change_protection(struct mmu_gather *tlb,
  			      struct vm_area_struct *vma, unsigned long start,
  			      unsigned long end, unsigned long cp_flags);
mm/mprotect.c +41 -12
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
 
  #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len)
  {
  	struct page *page;
+	bool temp_ret;
+	bool ret;
+	int i;
 
-	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-		return false;
+	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+		ret = false;
+		goto out;
+	}
 
  	/* Don't touch entries that are not even readable. */
-	if (pte_protnone(pte))
-		return false;
+	if (pte_protnone(pte)) {
+		ret = false;
+		goto out;
+	}
 
  	/* Do we need write faults for softdirty tracking? */
-	if (pte_needs_soft_dirty_wp(vma, pte))
-		return false;
+	if (pte_needs_soft_dirty_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
  	/* Do we need write faults for uffd-wp tracking? */
-	if (userfaultfd_pte_wp(vma, pte))
-		return false;
+	if (userfaultfd_pte_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
  	if (!(vma->vm_flags & VM_SHARED)) {
  		/*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  		 * any additional checks while holding the PT lock.
  		 */
  		page = vm_normal_page(vma, addr, pte);
-		return page && PageAnon(page) && PageAnonExclusive(page);
+		ret = (page && PageAnon(page) && PageAnonExclusive(page));
+		if (!len)
+			return ret;
+
+		/* Check how many consecutive pages are AnonExclusive or not */
+		for (i = 1; i < max_len; ++i) {
+			++page;
+			temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+			if (temp_ret != ret)
+				break;
+		}
+		*len = i;
+		return ret;
  	}
 
  	VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  	 * FS was already notified and we can simply mark the PTE writable
  	 * just like the write-fault handler would do.
  	 */
-	return pte_dirty(pte);
+	ret = pte_dirty(pte);
+
+out:
+	/* The entire batch is guaranteed to have the same return value */
+	if (len)
+		*len = max_len;
+	return ret;
  }
 
Quoted 7 lines

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 116

Quoted 3 lines
mm/mprotect.c +1 -1
  static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-		pte_t pte, int max_nr_ptes)
+		pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)

We'll almost certainly want more flags here in future. I wonder if it's cleaner just to pass "fpb_t extra_flags" here instead of the bool, then OR them in. You can pass 0 or FPB_IGNORE_SOFT_DIRTY at your 2 callsites. No strong opinion though.

I think your suggestion is better.

Quoted 96 lines
Quoted 96 lines
mm/mprotect.c +40 -5
  {
-	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	fpb_t flags = FPB_IGNORE_DIRTY;
 
-	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+	if (ignore_soft_dirty)
+		flags |= FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
  		return 1;
 
  	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
  			       NULL, NULL, NULL);
  }
 
+/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+		unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+		int max_len, int sub_batch_idx)
+{
+	unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+	pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+	pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+	pte_t *new_ptep = ptep + sub_batch_idx;
+	int len = 1;
+
+	if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+		new_ptent = pte_mkwrite(new_ptent, vma);
+
+	modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent, len);
+	if (pte_needs_flush(new_oldpte, new_ptent))
+		tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE);
+	return len;
+}
+
  static long change_pte_range(struct mmu_gather *tlb,
  		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
  		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
  	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
  	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-	int nr_ptes;
+	int sub_batch_idx, max_len, len, nr_ptes;
 
  	tlb_change_page_size(tlb, PAGE_SIZE);
  	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
  	flush_tlb_batched_pending(vma->vm_mm);
  	arch_enter_lazy_mmu_mode();
  	do {
+		sub_batch_idx = 0;
  		nr_ptes = 1;
  		oldpte = ptep_get(pte);
  		if (pte_present(oldpte)) {
  			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+			struct folio *folio = NULL;
  			pte_t ptent;
 
  			/*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
  			 * pages. See similar comment in change_huge_pmd.
  			 */
  			if (prot_numa) {
-				struct folio *folio;
  				int nid;
  				bool toptier;
 
@@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,
  				    toptier) {
  skip_batch:
  					nr_ptes = mprotect_batch(folio, addr, pte,
-								 oldpte, max_nr_ptes);
+								 oldpte, max_nr_ptes,
+								 true);
  					continue;
  				}
  				if (folio_use_access_time(folio))
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
  						jiffies_to_msecs(jiffies));
  			}
 
+			if (!folio)
+				folio = vm_normal_folio(vma, addr, oldpte);
+
+			nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+						 max_nr_ptes, false);
  			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
  			ptent = pte_modify(oldpte, newprot);
 
Quoted 8 lines

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 213

Quoted 6 lines
mm/mprotect.c +0 -2
@@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
  			 * example, if a PTE is already dirty and no other
  			 * COW or special handling is required.
  			 */
-			if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-			    !pte_write(ptent) &&

Don't you need to keep the !pte_write(ptent) condition here? No need to sub- batch if write is already set.

Ah right, the permissions for the entire batch are the same, so pte_write(ptent) will

return the same value for all ptes in the batch, so we can batch through this too. I

thought that we may skip on the wp-optimization for other ptes if the first

pte is writable and the others are not, but that won't happen because we are batching.

Quoted 72 lines
Quoted 14 lines
mm/mprotect.c +6 -7
-			    can_change_pte_writable(vma, addr, ptent))
-				ptent = pte_mkwrite(ptent, vma);
-
-			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-			if (pte_needs_flush(oldpte, ptent))
-				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-			pages++;
+			if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+				max_len = nr_ptes;
+				while (sub_batch_idx < nr_ptes) {
+
+					/* Get length of sub batch */
+					len = modify_sub_batch(vma, tlb, addr, pte,

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 291

Quoted 10 lines
mm/mprotect.c +10 -0
+							       oldpte, ptent, max_len,
+							       sub_batch_idx);
+					sub_batch_idx += len;
+					max_len -= len;
+				}
+			} else {
+				modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
+				if (pte_needs_flush(oldpte, ptent))
+					tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
+			}

This if/else block and modify_sub_block() is all pretty ugly. I wonder if something like this would be neater:

			use_sub_batch = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
					!pte_write(ptent) &&
					LIKE_can_change_pte_writable(vma, addr, ptent, folio);

			while (nr_ptes) {
				if (use_sub_batch)
					sub_nr_ptes = sub_batch(...);
				else
					sub_nr_ptes = nr_ptes;

				modify_prot_commit_ptes(vma, addr, pte, oldpte,
							ptent, sub_nr_ptes);
				if (pte_needs_flush(oldpte, ptent))
					tlb_flush_pte_range(tlb, addr,
						sub_nr_ptes * PAGE_SIZE);

				addr += sub_nr_ptes * PAGE_SIZE;
				pte += sub_nr_ptes;
				oldpte = pte_advance_pfn(oldpte, sub_nr_ptes);
				ptent = pte_advance_pfn(ptent, sub_nr_ptes);
				nr_ptes -= sub_nr_ptes;
				pages += sub_nr_ptes;
			}

Where:

- LIKE_can_change_pte_writable() is similar to can_change_pte_writable() but does everything except checking the per-page exclusive flag. Note that we already have the folio so can pass that rather than calling vm_normal_page() again.

- sub_batch() can be passed the folio and the index of the first page within the folio and the max number of pages to check (nr_ptes). Then returns the number of pages where exclusive flag is the same.

Obviously they both need better names...

Thanks,
Ryan

Quoted 4 lines
mm/mprotect.c +1 -0
+			pages += nr_ptes;
  		} else if (is_swap_pte(oldpte)) {
  			swp_entry_t entry = pte_to_swp_entry(oldpte);
  			pte_t newpte;
author
#24 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

In reply to Ryan Roberts
Quoted 312 lines · Ryan Roberts
Quoted 4 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).

parisc's pte_needs_flush() is a different (static) function, just coincidentally named - it takes different arguments.

But powerpc and x86 both specialize it and they consider the dirty bit. Both implementations look to me like they will function correctly but it is a bit ugly. They both conclude that if there is no change or if dirty has gone from clear to set, then no flush is needed. (flush only needed when dirty goes from set to clear). Given after your change, oldpte may have dirty set when it wasn't really set for the pte in question that means the function could detect no change when really its a clear -> set change; it gives the same answer in both cases so it's safe. Perhaps a bit fragile though...

Quoted 7 lines

For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is),

Quoted 8 lines

the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same PageAnonExclusive condition, and process that sub batch, then determine and process the next sub batch, and so on. Note that this does not cause any extra overhead; if suppose the size of the folio batch is 512, then the sub batch processing in total will take 512 iterations, which is the same as what we would have done before.

This commit log could do with some reformatting; blank lines between paragraphs and break at whatever the usual git commit log char limit is (72 chars?).

Quoted 11 lines
---
  include/linux/mm.h |   7 ++-
  mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
  2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +0 -0
index 43748c8f3454..7d5b96f005dc 100644

include/linux/mm.h int get_cmdline(struct task_struct *task, char *buffer, int buflen); · line 2548

Quoted 9 lines
include/linux/mm.h +4 -2
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
  #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
  					    MM_CP_UFFD_WP_RESOLVE)
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)	\
+	can_change_ptes_writable(vma, addr, pte, 1, NULL)

nit: add an extra tab for readability:

#define can_change_pte_writable(vma, addr, pte) \
can_change_ptes_writable(vma, addr, pte, 1, NULL)

Quoted 83 lines
include/linux/mm.h +1 -0
+
  extern long change_protection(struct mmu_gather *tlb,
  			      struct vm_area_struct *vma, unsigned long start,
  			      unsigned long end, unsigned long cp_flags);
mm/mprotect.c +36 -12
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
 
  #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len)
  {
  	struct page *page;
+	bool temp_ret;
+	bool ret;
+	int i;
 
-	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-		return false;
+	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+		ret = false;
+		goto out;
+	}
 
  	/* Don't touch entries that are not even readable. */
-	if (pte_protnone(pte))
-		return false;
+	if (pte_protnone(pte)) {
+		ret = false;
+		goto out;
+	}
 
  	/* Do we need write faults for softdirty tracking? */
-	if (pte_needs_soft_dirty_wp(vma, pte))
-		return false;
+	if (pte_needs_soft_dirty_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
  	/* Do we need write faults for uffd-wp tracking? */
-	if (userfaultfd_pte_wp(vma, pte))
-		return false;
+	if (userfaultfd_pte_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
  	if (!(vma->vm_flags & VM_SHARED)) {
  		/*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  		 * any additional checks while holding the PT lock.
  		 */
  		page = vm_normal_page(vma, addr, pte);
-		return page && PageAnon(page) && PageAnonExclusive(page);
+		ret = (page && PageAnon(page) && PageAnonExclusive(page));
+		if (!len)
+			return ret;
+
+		/* Check how many consecutive pages are AnonExclusive or not */
+		for (i = 1; i < max_len; ++i) {
+			++page;
+			temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+			if (temp_ret != ret)
+				break;
+		}
+		*len = i;
+		return ret;
  	}
 
  	VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  	 * FS was already notified and we can simply mark the PTE writable
  	 * just like the write-fault handler would do.
  	 */
-	return pte_dirty(pte);
+	ret = pte_dirty(pte);
+

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 116

Quoted 10 lines
mm/mprotect.c +6 -1
+out:
+	/* The entire batch is guaranteed to have the same return value */
+	if (len)
+		*len = max_len;
+	return ret;
  }
 
  static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-		pte_t pte, int max_nr_ptes)
+		pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)

We'll almost certainly want more flags here in future. I wonder if it's cleaner just to pass "fpb_t extra_flags" here instead of the bool, then OR them in. You can pass 0 or FPB_IGNORE_SOFT_DIRTY at your 2 callsites. No strong opinion though.

Quoted 96 lines
mm/mprotect.c +40 -5
  {
-	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	fpb_t flags = FPB_IGNORE_DIRTY;
 
-	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+	if (ignore_soft_dirty)
+		flags |= FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
  		return 1;
 
  	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
  			       NULL, NULL, NULL);
  }
 
+/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+		unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+		int max_len, int sub_batch_idx)
+{
+	unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+	pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+	pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+	pte_t *new_ptep = ptep + sub_batch_idx;
+	int len = 1;
+
+	if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+		new_ptent = pte_mkwrite(new_ptent, vma);
+
+	modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent, len);
+	if (pte_needs_flush(new_oldpte, new_ptent))
+		tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE);
+	return len;
+}
+
  static long change_pte_range(struct mmu_gather *tlb,
  		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
  		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
  	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
  	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-	int nr_ptes;
+	int sub_batch_idx, max_len, len, nr_ptes;
 
  	tlb_change_page_size(tlb, PAGE_SIZE);
  	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
  	flush_tlb_batched_pending(vma->vm_mm);
  	arch_enter_lazy_mmu_mode();
  	do {
+		sub_batch_idx = 0;
  		nr_ptes = 1;
  		oldpte = ptep_get(pte);
  		if (pte_present(oldpte)) {
  			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+			struct folio *folio = NULL;
  			pte_t ptent;
 
  			/*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
  			 * pages. See similar comment in change_huge_pmd.
  			 */
  			if (prot_numa) {
-				struct folio *folio;
  				int nid;
  				bool toptier;
 
@@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,
  				    toptier) {
  skip_batch:
  					nr_ptes = mprotect_batch(folio, addr, pte,
-								 oldpte, max_nr_ptes);
+								 oldpte, max_nr_ptes,
+								 true);
  					continue;
  				}
  				if (folio_use_access_time(folio))
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
  						jiffies_to_msecs(jiffies));
  			}
 
+			if (!folio)
+				folio = vm_normal_folio(vma, addr, oldpte);
+
+			nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+						 max_nr_ptes, false);
  			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
  			ptent = pte_modify(oldpte, newprot);
 

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 213

Quoted 5 lines
mm/mprotect.c +0 -2
@@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
  			 * example, if a PTE is already dirty and no other
  			 * COW or special handling is required.
  			 */
-			if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-			    !pte_write(ptent) &&

Don't you need to keep the !pte_write(ptent) condition here? No need to sub- batch if write is already set.

Quoted 13 lines
mm/mprotect.c +6 -7
-			    can_change_pte_writable(vma, addr, ptent))
-				ptent = pte_mkwrite(ptent, vma);
-
-			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-			if (pte_needs_flush(oldpte, ptent))
-				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-			pages++;
+			if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+				max_len = nr_ptes;
+				while (sub_batch_idx < nr_ptes) {
+
+					/* Get length of sub batch */
+					len = modify_sub_batch(vma, tlb, addr, pte,

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 291

Quoted 10 lines
mm/mprotect.c +10 -0
+							       oldpte, ptent, max_len,
+							       sub_batch_idx);
+					sub_batch_idx += len;
+					max_len -= len;
+				}
+			} else {
+				modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
+				if (pte_needs_flush(oldpte, ptent))
+					tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
+			}

This if/else block and modify_sub_block() is all pretty ugly. I wonder if something like this would be neater:

			use_sub_batch = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
					!pte_write(ptent) &&
					LIKE_can_change_pte_writable(vma, addr, ptent, folio);

			while (nr_ptes) {
				if (use_sub_batch)
					sub_nr_ptes = sub_batch(...);
				else
					sub_nr_ptes = nr_ptes;

				modify_prot_commit_ptes(vma, addr, pte, oldpte,
							ptent, sub_nr_ptes);
				if (pte_needs_flush(oldpte, ptent))
					tlb_flush_pte_range(tlb, addr,
						sub_nr_ptes * PAGE_SIZE);

				addr += sub_nr_ptes * PAGE_SIZE;
				pte += sub_nr_ptes;
				oldpte = pte_advance_pfn(oldpte, sub_nr_ptes);
				ptent = pte_advance_pfn(ptent, sub_nr_ptes);
				nr_ptes -= sub_nr_ptes;
				pages += sub_nr_ptes;
			}
Quoted 6 lines

Where:

- LIKE_can_change_pte_writable() is similar to can_change_pte_writable() but does everything except checking the per-page exclusive flag. Note that we already have the folio so can pass that rather than calling vm_normal_page() again.

Hmm...all I can say is that I can *try* to come up with something cleaner, but can't

promise :) I'll try modifying your approach.

Quoted 15 lines

- sub_batch() can be passed the folio and the index of the first page within the folio and the max number of pages to check (nr_ptes). Then returns the number of pages where exclusive flag is the same.

Obviously they both need better names...

Thanks,
Ryan

Quoted 4 lines
mm/mprotect.c +1 -0
+			pages += nr_ptes;
  		} else if (is_swap_pte(oldpte)) {
  			swp_entry_t entry = pte_to_swp_entry(oldpte);
  			pte_t newpte;
author
#25 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

In reply to Ryan Roberts
Quoted 319 lines · Ryan Roberts
Quoted 4 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).

parisc's pte_needs_flush() is a different (static) function, just coincidentally named - it takes different arguments.

But powerpc and x86 both specialize it and they consider the dirty bit. Both implementations look to me like they will function correctly but it is a bit ugly. They both conclude that if there is no change or if dirty has gone from clear to set, then no flush is needed. (flush only needed when dirty goes from set to clear). Given after your change, oldpte may have dirty set when it wasn't really set for the pte in question that means the function could detect no change when really its a clear -> set change; it gives the same answer in both cases so it's safe. Perhaps a bit fragile though...

Quoted 7 lines

For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is),

Quoted 8 lines

the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same PageAnonExclusive condition, and process that sub batch, then determine and process the next sub batch, and so on. Note that this does not cause any extra overhead; if suppose the size of the folio batch is 512, then the sub batch processing in total will take 512 iterations, which is the same as what we would have done before.

This commit log could do with some reformatting; blank lines between paragraphs and break at whatever the usual git commit log char limit is (72 chars?).

Quoted 11 lines
---
  include/linux/mm.h |   7 ++-
  mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
  2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +0 -0
index 43748c8f3454..7d5b96f005dc 100644

include/linux/mm.h int get_cmdline(struct task_struct *task, char *buffer, int buflen); · line 2548

Quoted 9 lines
include/linux/mm.h +4 -2
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
  #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
  					    MM_CP_UFFD_WP_RESOLVE)
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)	\
+	can_change_ptes_writable(vma, addr, pte, 1, NULL)

nit: add an extra tab for readability:

#define can_change_pte_writable(vma, addr, pte) \
can_change_ptes_writable(vma, addr, pte, 1, NULL)

Quoted 83 lines
include/linux/mm.h +1 -0
+
  extern long change_protection(struct mmu_gather *tlb,
  			      struct vm_area_struct *vma, unsigned long start,
  			      unsigned long end, unsigned long cp_flags);
mm/mprotect.c +36 -12
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
 
  #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte, int max_len, int *len)
  {
  	struct page *page;
+	bool temp_ret;
+	bool ret;
+	int i;
 
-	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-		return false;
+	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+		ret = false;
+		goto out;
+	}
 
  	/* Don't touch entries that are not even readable. */
-	if (pte_protnone(pte))
-		return false;
+	if (pte_protnone(pte)) {
+		ret = false;
+		goto out;
+	}
 
  	/* Do we need write faults for softdirty tracking? */
-	if (pte_needs_soft_dirty_wp(vma, pte))
-		return false;
+	if (pte_needs_soft_dirty_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
  	/* Do we need write faults for uffd-wp tracking? */
-	if (userfaultfd_pte_wp(vma, pte))
-		return false;
+	if (userfaultfd_pte_wp(vma, pte)) {
+		ret = false;
+		goto out;
+	}
 
  	if (!(vma->vm_flags & VM_SHARED)) {
  		/*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  		 * any additional checks while holding the PT lock.
  		 */
  		page = vm_normal_page(vma, addr, pte);
-		return page && PageAnon(page) && PageAnonExclusive(page);
+		ret = (page && PageAnon(page) && PageAnonExclusive(page));
+		if (!len)
+			return ret;
+
+		/* Check how many consecutive pages are AnonExclusive or not */
+		for (i = 1; i < max_len; ++i) {
+			++page;
+			temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+			if (temp_ret != ret)
+				break;
+		}
+		*len = i;
+		return ret;
  	}
 
  	VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
  	 * FS was already notified and we can simply mark the PTE writable
  	 * just like the write-fault handler would do.
  	 */
-	return pte_dirty(pte);
+	ret = pte_dirty(pte);
+

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, · line 116

Quoted 10 lines
mm/mprotect.c +6 -1
+out:
+	/* The entire batch is guaranteed to have the same return value */
+	if (len)
+		*len = max_len;
+	return ret;
  }
 
  static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-		pte_t pte, int max_nr_ptes)
+		pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)

We'll almost certainly want more flags here in future. I wonder if it's cleaner just to pass "fpb_t extra_flags" here instead of the bool, then OR them in. You can pass 0 or FPB_IGNORE_SOFT_DIRTY at your 2 callsites. No strong opinion though.

Quoted 96 lines
mm/mprotect.c +40 -5
  {
-	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	fpb_t flags = FPB_IGNORE_DIRTY;
 
-	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+	if (ignore_soft_dirty)
+		flags |= FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
  		return 1;
 
  	return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
  			       NULL, NULL, NULL);
  }
 
+/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+		unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+		int max_len, int sub_batch_idx)
+{
+	unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+	pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+	pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+	pte_t *new_ptep = ptep + sub_batch_idx;
+	int len = 1;
+
+	if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+		new_ptent = pte_mkwrite(new_ptent, vma);
+
+	modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent, len);
+	if (pte_needs_flush(new_oldpte, new_ptent))
+		tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE);
+	return len;
+}
+
  static long change_pte_range(struct mmu_gather *tlb,
  		struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
  		unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
  	bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
  	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
  	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-	int nr_ptes;
+	int sub_batch_idx, max_len, len, nr_ptes;
 
  	tlb_change_page_size(tlb, PAGE_SIZE);
  	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
  	flush_tlb_batched_pending(vma->vm_mm);
  	arch_enter_lazy_mmu_mode();
  	do {
+		sub_batch_idx = 0;
  		nr_ptes = 1;
  		oldpte = ptep_get(pte);
  		if (pte_present(oldpte)) {
  			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+			struct folio *folio = NULL;
  			pte_t ptent;
 
  			/*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
  			 * pages. See similar comment in change_huge_pmd.
  			 */
  			if (prot_numa) {
-				struct folio *folio;
  				int nid;
  				bool toptier;
 
@@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,
  				    toptier) {
  skip_batch:
  					nr_ptes = mprotect_batch(folio, addr, pte,
-								 oldpte, max_nr_ptes);
+								 oldpte, max_nr_ptes,
+								 true);
  					continue;
  				}
  				if (folio_use_access_time(folio))
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
  						jiffies_to_msecs(jiffies));
  			}
 
+			if (!folio)
+				folio = vm_normal_folio(vma, addr, oldpte);
+
+			nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+						 max_nr_ptes, false);
  			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
  			ptent = pte_modify(oldpte, newprot);
 

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 213

Quoted 5 lines
mm/mprotect.c +0 -2
@@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
  			 * example, if a PTE is already dirty and no other
  			 * COW or special handling is required.
  			 */
-			if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-			    !pte_write(ptent) &&

Don't you need to keep the !pte_write(ptent) condition here? No need to sub- batch if write is already set.

Quoted 13 lines
mm/mprotect.c +6 -7
-			    can_change_pte_writable(vma, addr, ptent))
-				ptent = pte_mkwrite(ptent, vma);
-
-			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-			if (pte_needs_flush(oldpte, ptent))
-				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-			pages++;
+			if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+				max_len = nr_ptes;
+				while (sub_batch_idx < nr_ptes) {
+
+					/* Get length of sub batch */
+					len = modify_sub_batch(vma, tlb, addr, pte,

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb, · line 291

Quoted 10 lines
mm/mprotect.c +10 -0
+							       oldpte, ptent, max_len,
+							       sub_batch_idx);
+					sub_batch_idx += len;
+					max_len -= len;
+				}
+			} else {
+				modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
+				if (pte_needs_flush(oldpte, ptent))
+					tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE);
+			}

This if/else block and modify_sub_block() is all pretty ugly. I wonder if something like this would be neater:

			use_sub_batch = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
					!pte_write(ptent) &&
					LIKE_can_change_pte_writable(vma, addr, ptent, folio);

			while (nr_ptes) {
				if (use_sub_batch)
					sub_nr_ptes = sub_batch(...);
				else
					sub_nr_ptes = nr_ptes;

				modify_prot_commit_ptes(vma, addr, pte, oldpte,
							ptent, sub_nr_ptes);
				if (pte_needs_flush(oldpte, ptent))
					tlb_flush_pte_range(tlb, addr,
						sub_nr_ptes * PAGE_SIZE);

				addr += sub_nr_ptes * PAGE_SIZE;
				pte += sub_nr_ptes;
				oldpte = pte_advance_pfn(oldpte, sub_nr_ptes);
				ptent = pte_advance_pfn(ptent, sub_nr_ptes);
				nr_ptes -= sub_nr_ptes;
				pages += sub_nr_ptes;
			}

Where:

- LIKE_can_change_pte_writable() is similar to can_change_pte_writable() but does everything except checking the per-page exclusive flag. Note that we already have the folio so can pass that rather than calling vm_normal_page() again.

Quoted 5 lines

- sub_batch() can be passed the folio and the index of the first page within the folio and the max number of pages to check (nr_ptes). Then returns the number of pages where exclusive flag is the same.

Obviously they both need better names...

I cannot figure a nice way of implementing your suggestion. We need to get the length of the sub batch and the true/false return value from can_change_ptes_writable, and we also need to call pte_mkwrite in case the ret is true. Doing that in your suggested way seems harder to me, along with the fact that it is causing indentation problem by one extra tab compared to my current implementation.

I don't know, my method looks pretty neat to me : )

Quoted 9 lines

Thanks,
Ryan

Quoted 4 lines
mm/mprotect.c +1 -0
+			pages += nr_ptes;
  		} else if (is_swap_pte(oldpte)) {
  			swp_entry_t entry = pte_to_swp_entry(oldpte);
  			pte_t newpte;
#26 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

Quoted 331 lines · Dev Jain
Quoted 314 lines · Ryan Roberts
Quoted 5 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa
case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).

parisc's pte_needs_flush() is a different (static) function, just coincidentally named - it takes different arguments.

But powerpc and x86 both specialize it and they consider the dirty bit. Both implementations look to me like they will function correctly but it is a bit ugly. They both conclude that if there is no change or if dirty has gone from clear to set, then no flush is needed. (flush only needed when dirty goes from set to clear). Given after your change, oldpte may have dirty set when it wasn't really set for the pte in question that means the function could detect no change when really its a clear -> set change; it gives the same answer in both cases so it's safe. Perhaps a bit fragile though...

Quoted 13 lines

For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits
across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is),
the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same

Quoted 8 lines

PageAnonExclusive
condition, and process that sub batch, then determine and process the next sub batch,
and so on. Note that this does not cause any extra overhead; if suppose the size of
the folio batch is 512, then the sub batch processing in total will take 512 iterations,
which is the same as what we would have done before.

This commit log could do with some reformatting; blank lines between paragraphs and break at whatever the usual git commit log char limit is (72 chars?).

Quoted 11 lines

---
  include/linux/mm.h |   7 ++-
  mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------

  2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +0 -0
index 43748c8f3454..7d5b96f005dc 100644

include/linux/mm.h int get_cmdline(struct task_struct *task, char · line 2549

Quoted 9 lines
include/linux/mm.h +4 -1
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char
 *buffer, int buflen);
   #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
                           MM_CP_UFFD_WP_RESOLVE)
   -bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                 pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                 pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)    \
+    can_change_ptes_writable(vma, addr, pte, 1, NULL)

nit: add an extra tab for readability:

#define can_change_pte_writable(vma, addr, pte)    \
        can_change_ptes_writable(vma, addr, pte, 1, NULL)

Quoted 83 lines
include/linux/mm.h +1 -0
+
   extern long change_protection(struct mmu_gather *tlb,
                     struct vm_area_struct *vma, unsigned long start,

                    unsigned long end, unsigned long cp_flags);

mm/mprotect.c +41 -10
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
     #include "internal.h"
   -bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                 pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                 pte_t pte, int max_len, int *len)
   {
       struct page *page;
+    bool temp_ret;
+    bool ret;
+    int i;
   -    if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-        return false;
+    if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+        ret = false;
+        goto out;
+    }
         /* Don't touch entries that are not even readable. */
-    if (pte_protnone(pte))
-        return false;
+    if (pte_protnone(pte)) {
+        ret = false;
+        goto out;
+    }
         /* Do we need write faults for softdirty tracking? */
-    if (pte_needs_soft_dirty_wp(vma, pte))
-        return false;
+    if (pte_needs_soft_dirty_wp(vma, pte)) {
+        ret = false;
+        goto out;
+    }
         /* Do we need write faults for uffd-wp tracking? */
-    if (userfaultfd_pte_wp(vma, pte))
-        return false;
+    if (userfaultfd_pte_wp(vma, pte)) {
+        ret = false;
+        goto out;
+    }
         if (!(vma->vm_flags & VM_SHARED)) {
           /*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma,
 unsigned long addr,
            * any additional checks while holding the PT lock.
            */
           page = vm_normal_page(vma, addr, pte);
-        return page && PageAnon(page) && PageAnonExclusive(page);
+        ret = (page && PageAnon(page) && PageAnonExclusive(page));
+        if (!len)
+            return ret;
+
+        /* Check how many consecutive pages are AnonExclusive or not */
+        for (i = 1; i < max_len; ++i) {
+            ++page;
+            temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+            if (temp_ret != ret)
+                break;
+        }
+        *len = i;
+        return ret;
       }
         VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma,
 unsigned long addr,
        * FS was already notified and we can simply mark the PTE writable
        * just like the write-fault handler would do.
        */
-    return pte_dirty(pte);
+    ret = pte_dirty(pte);
+
+out:
+    /* The entire batch is guaranteed to have the same return value */
+    if (len)
+        *len = max_len;
+    return ret;
   }
Quoted 4 lines

    static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-        pte_t pte, int max_nr_ptes)
+        pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)

We'll almost certainly want more flags here in future. I wonder if it's cleaner just to pass "fpb_t extra_flags" here instead of the bool, then OR them in. You can pass 0 or FPB_IGNORE_SOFT_DIRTY at your 2 callsites. No strong opinion though.

Quoted 89 lines

  {
-    const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; +    fpb_t flags = FPB_IGNORE_DIRTY;
  -    if (!folio_test_large(folio) || (max_nr_ptes == 1))
+    if (ignore_soft_dirty)
+        flags |= FPB_IGNORE_SOFT_DIRTY;
+
+    if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1)) return 1;
        return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags, NULL, NULL, NULL);
  }
  +/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */

mm/mprotect.c +14 -0
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+        unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+        int max_len, int sub_batch_idx)
+{
+    unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+    pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+    pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+    pte_t *new_ptep = ptep + sub_batch_idx;
+    int len = 1;
+
+    if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+        new_ptent = pte_mkwrite(new_ptent, vma);
+
+    modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent,

len);
+    if (pte_needs_flush(new_oldpte, new_ptent))
+        tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE); +    return len;

mm/mprotect.c +2 -0
+}
+

  static long change_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, unsigned long cp_flags)

mm/mprotect.c +3 -2
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
       bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
       bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
       bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-    int nr_ptes;
+    int sub_batch_idx, max_len, len, nr_ptes;
         tlb_change_page_size(tlb, PAGE_SIZE);
       pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
       flush_tlb_batched_pending(vma->vm_mm);
       arch_enter_lazy_mmu_mode();
       do {
+        sub_batch_idx = 0;
           nr_ptes = 1;
           oldpte = ptep_get(pte);
           if (pte_present(oldpte)) {
               int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+            struct folio *folio = NULL;
               pte_t ptent;
                 /*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
                * pages. See similar comment in change_huge_pmd.
                */
               if (prot_numa) {
-                struct folio *folio;
                   int nid;
                   bool toptier;
   @@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,

                      toptier) {
  skip_batch:
                      nr_ptes = mprotect_batch(folio, addr, pte, -                                 oldpte, max_nr_ptes); +                                 oldpte, max_nr_ptes, +                                 true);
                      continue;
                  }
                  if (folio_use_access_time(folio))

mm/mprotect.c +4 -0
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
                           jiffies_to_msecs(jiffies));
               }
   +            if (!folio)
+                folio = vm_normal_folio(vma, addr, oldpte);
+
+            nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+                         max_nr_ptes, false);
               oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
               ptent = pte_modify(oldpte, newprot);
   @@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
Quoted 5 lines

               * example, if a PTE is already dirty and no other
               * COW or special handling is required. */
-            if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && -                !pte_write(ptent) &&

Don't you need to keep the !pte_write(ptent) condition here? No need to sub- batch if write is already set.

Quoted 20 lines

-                can_change_pte_writable(vma, addr, ptent)) -                ptent = pte_mkwrite(ptent, vma);

mm/mprotect.c +13 -5
-
-            modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-            if (pte_needs_flush(oldpte, ptent))
-                tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-            pages++;
+            if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+                max_len = nr_ptes;
+                while (sub_batch_idx < nr_ptes) {
+
+                    /* Get length of sub batch */
+                    len = modify_sub_batch(vma, tlb, addr, pte,
+                                   oldpte, ptent, max_len,
+                                   sub_batch_idx);
+                    sub_batch_idx += len;
+                    max_len -= len;
+                }
+            } else {
+                modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent,
Quoted 4 lines

nr_ptes);
+                if (pte_needs_flush(oldpte, ptent))
+                    tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE); +            }

This if/else block and modify_sub_block() is all pretty ugly. I wonder if something like this would be neater:

            use_sub_batch = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pte_write(ptent) && LIKE_can_change_pte_writable(vma, addr, ptent, folio);

            while (nr_ptes) {
                if (use_sub_batch)
                    sub_nr_ptes = sub_batch(...);
                else
                    sub_nr_ptes = nr_ptes;

                modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, sub_nr_ptes);
                if (pte_needs_flush(oldpte, ptent)) tlb_flush_pte_range(tlb, addr, sub_nr_ptes * PAGE_SIZE);

                addr += sub_nr_ptes * PAGE_SIZE;
                pte += sub_nr_ptes;
                oldpte = pte_advance_pfn(oldpte, sub_nr_ptes); ptent = pte_advance_pfn(ptent, sub_nr_ptes); nr_ptes -= sub_nr_ptes;
                pages += sub_nr_ptes;
            }

Where:

  - LIKE_can_change_pte_writable() is similar to can_change_pte_writable() but does everything except checking the per-page exclusive flag. Note that we already have the folio so can pass that rather than calling vm_normal_page() again.

Quoted 5 lines

  - sub_batch() can be passed the folio and the index of the first page within the folio and the max number of pages to check (nr_ptes). Then returns the number of pages where exclusive flag is the same.

Obviously they both need better names...

I cannot figure a nice way of implementing your suggestion. We need to get the length
of the sub batch and the true/false return value from can_change_ptes_writable, and we also need to call pte_mkwrite in case the ret is true. Doing that in your suggested
way seems harder to me, along with the fact that it is causing indentation problem by
one extra tab compared to my current implementation.

Quoted 1 line

I don't know, my method looks pretty neat to me : )

How about something like this (compile-tested only)? Personally I find this easier to follow:

---8<---

mm/mprotect.c +23 -11
index 124612ce3d24..97ccc2a92a7b 100644
@@ -40,35 +40,47 @@
 
 #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte)
-{
-	struct page *page;
+enum tristate {
+	TRI_FALSE = 0,
+	TRI_TRUE = 1,
+	TRI_MAYBE = -1,
+};
 
+/*
+ * Returns enum tristate indicating whether the pte can be changed to writable.
+ * If TRI_MAYBE is returned, then the folio is anonymous and the user must
+ * additionally check PageAnonExclusive() for every page in the desired range.
+ */
+static int maybe_change_pte_writable(struct vm_area_struct *vma,
+				     unsigned long addr, pte_t pte,
+				     struct folio *folio)
+{
 	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-		return false;
+		return TRI_FALSE;
 
 	/* Don't touch entries that are not even readable. */
 	if (pte_protnone(pte))
-		return false;
+		return TRI_FALSE;
 
 	/* Do we need write faults for softdirty tracking? */
 	if (pte_needs_soft_dirty_wp(vma, pte))
-		return false;
+		return TRI_FALSE;
 
 	/* Do we need write faults for uffd-wp tracking? */
 	if (userfaultfd_pte_wp(vma, pte))
-		return false;
+		return TRI_FALSE;
 
 	if (!(vma->vm_flags & VM_SHARED)) {
 		/*
 		 * Writable MAP_PRIVATE mapping: We can only special-case on
 		 * exclusive anonymous pages, because we know that our
 		 * write-fault handler similarly would map them writable without
-		 * any additional checks while holding the PT lock.
+		 * any additional checks while holding the PT lock. So if the
+		 * folio is not anonymous, we know we cannot change pte
+		 * writable. If it is anonymous then the caller must further
+		 * check that the page is AnonExclusive().
 		 */
-		page = vm_normal_page(vma, addr, pte);
-		return page && PageAnon(page) && PageAnonExclusive(page);
+		return (!folio || folio_test_anon(folio)) ? TRI_MAYBE : TRI_FALSE;
 	}
 
 	VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,15 +92,60 @@ bool can_change_pte_writable(struct vm_area_struct *vma,

unsigned long addr,
* FS was already notified and we can simply mark the PTE writable
* just like the write-fault handler would do.
*/
- return pte_dirty(pte);
+ return pte_dirty(pte) ? TRI_TRUE : TRI_FALSE;
+}
+
+/*
+ * Returns the number of pages within the folio, starting from the page
+ * indicated by pgidx and up to pgidx + max_nr, that have the same value of
+ * PageAnonExclusive(). Must only be called for anonymous folios. Value of
+ * PageAnonExclusive() is returned in *exclusive.
+ */
+static int anon_exclusive_batch(struct folio *folio, int pgidx, int max_nr,
+ bool *exclusive)
+{
+ VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
+ VM_WARN_ON_FOLIO(folio_nr_pages(folio) < pgidx + max_nr, folio);
+
+ struct page *page = folio_page(folio, pgidx++);
+ int nr = 1;
+
+ *exclusive = PageAnonExclusive(page);
+
+ while (nr < max_nr) {
+ page = folio_page(folio, pgidx++);
+ if (*exclusive != PageAnonExclusive(page))
+ break;
+ nr++;
+ }
+
+ return nr;
+}
+
+bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
+ pte_t pte)
+{
+ int ret;
+ struct page *page;
+
+ ret = maybe_change_pte_writable(vma, addr, pte, NULL);
+ if (ret == TRI_MAYBE) {
+ page = vm_normal_page(vma, addr, pte);
+ ret = page && PageAnon(page) && PageAnonExclusive(page);
+ }
+
+ return ret;
}

static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
- pte_t pte, int max_nr_ptes)
+ pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)
{
- const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+ fpb_t flags = FPB_IGNORE_DIRTY;

- if (!folio_test_large(folio) || (max_nr_ptes == 1))
+ if (ignore_soft_dirty)
+ flags |= FPB_IGNORE_SOFT_DIRTY;
+
+ if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
return 1;

return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
@@ -125,14 +182,17 @@ static long change_pte_range(struct mmu_gather *tlb,
oldpte = ptep_get(pte);
if (pte_present(oldpte)) {
int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
- pte_t ptent;
+ struct folio *folio = NULL;
+ int sub_nr_ptes, pgidx;
+ pte_t ptent, newpte;
+ bool sub_set_write;
+ int set_write;

/*
* Avoid trapping faults against the zero or KSM
* pages. See similar comment in change_huge_pmd. */
if (prot_numa) {
- struct folio *folio;
int nid;
bool toptier;

@@ -180,7 +240,8 @@ static long change_pte_range(struct mmu_gather *tlb,
toptier) {
skip_batch:
nr_ptes = mprotect_batch(folio, addr, pte,
- oldpte, max_nr_ptes);
+ oldpte, max_nr_ptes,
+ true);
continue;
}
if (folio_use_access_time(folio))
@@ -188,6 +249,11 @@ static long change_pte_range(struct mmu_gather *tlb,
jiffies_to_msecs(jiffies));
}

+ if (!folio)
+ folio = vm_normal_folio(vma, addr, oldpte);
+
+ nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+ max_nr_ptes, false);
oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes); ptent = pte_modify(oldpte, newprot);

@@ -209,15 +275,38 @@ static long change_pte_range(struct mmu_gather *tlb,
* example, if a PTE is already dirty and no other
* COW or special handling is required. */
- if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
- !pte_write(ptent) &&
- can_change_pte_writable(vma, addr, ptent))
- ptent = pte_mkwrite(ptent, vma);
-
- modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
- if (pte_needs_flush(oldpte, ptent))
- tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
- pages++;
+ set_write = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
+ !pte_write(ptent) &&
+ maybe_change_pte_writable(vma, addr, ptent, folio);
+
+ while (nr_ptes) {
+ if (set_write == TRI_MAYBE) {
+ pgidx = folio_pfn(folio) - pte_pfn(ptent);
+ sub_nr_ptes = anon_exclusive_batch(folio,
+ pgidx, nr_ptes, &sub_set_write);
+ } else {
+ sub_nr_ptes = nr_ptes;
+ sub_set_write = set_write == TRI_TRUE;
+ }
+
+ if (sub_set_write)
+ newpte = pte_mkwrite(ptent, vma);
+ else
+ newpte = ptent;
+
+ modify_prot_commit_ptes(vma, addr, pte, oldpte,
+ newpte, sub_nr_ptes);
+ if (pte_needs_flush(oldpte, newpte))
+ tlb_flush_pte_range(tlb, addr,
+ sub_nr_ptes * PAGE_SIZE);
+
+ addr += sub_nr_ptes * PAGE_SIZE;
+ pte += sub_nr_ptes;
+ oldpte = pte_advance_pfn(oldpte, sub_nr_ptes);
+ ptent = pte_advance_pfn(ptent, sub_nr_ptes);
+ nr_ptes -= sub_nr_ptes;
+ pages += sub_nr_ptes;
+ }
} else if (is_swap_pte(oldpte)) {
swp_entry_t entry = pte_to_swp_entry(oldpte); pte_t newpte;
---8<---

Thanks,
Ryan

author
#27 lore raw

Re: [PATCH v3 3/5] mm: Optimize mprotect() by PTE batching

Quoted 546 lines · Ryan Roberts
Quoted 329 lines · Dev Jain
Quoted 314 lines · Ryan Roberts
Quoted 5 lines · Dev Jain

Use folio_pte_batch to batch process a large folio. Reuse the folio from prot_numa
case if possible. Since modify_prot_start_ptes() gathers access/dirty bits, it lets us batch around pte_needs_flush() (for parisc, the definition includes the access bit).

parisc's pte_needs_flush() is a different (static) function, just coincidentally named - it takes different arguments.

But powerpc and x86 both specialize it and they consider the dirty bit. Both implementations look to me like they will function correctly but it is a bit ugly. They both conclude that if there is no change or if dirty has gone from clear to set, then no flush is needed. (flush only needed when dirty goes from set to clear). Given after your change, oldpte may have dirty set when it wasn't really set for the pte in question that means the function could detect no change when really its a clear -> set change; it gives the same answer in both cases so it's safe. Perhaps a bit fragile though...

Quoted 13 lines

For all cases other than the PageAnonExclusive case, if the case holds true for one pte in the batch, one can confirm that that case will hold true for other ptes in the batch too; for pte_needs_soft_dirty_wp(), we do not pass FPB_IGNORE_SOFT_DIRTY. modify_prot_start_ptes() collects the dirty and access bits
across the batch, therefore batching across pte_dirty(): this is correct since the dirty bit on the PTE really is just an indication that the folio got written to, so even if the PTE is not actually dirty (but one of the PTEs in the batch is),
the wp-fault optimization can be made.
The crux now is how to batch around the PageAnonExclusive case; we must check the corresponding condition for every single page. Therefore, from the large folio batch, we process sub batches of ptes mapping pages with the same

Quoted 8 lines

PageAnonExclusive
condition, and process that sub batch, then determine and process the next sub batch,
and so on. Note that this does not cause any extra overhead; if suppose the size of
the folio batch is 512, then the sub batch processing in total will take 512 iterations,
which is the same as what we would have done before.

This commit log could do with some reformatting; blank lines between paragraphs and break at whatever the usual git commit log char limit is (72 chars?).

Quoted 11 lines
---
   include/linux/mm.h |   7 ++-
   mm/mprotect.c      | 126 +++++++++++++++++++++++++++++++++++----------
   2 files changed, 104 insertions(+), 29 deletions(-)
include/linux/mm.h +0 -0
index 43748c8f3454..7d5b96f005dc 100644

include/linux/mm.h int get_cmdline(struct task_struct *task, char · line 2549

Quoted 9 lines
include/linux/mm.h +4 -1
@@ -2542,8 +2542,11 @@ int get_cmdline(struct task_struct *task, char
 *buffer, int buflen);
   #define  MM_CP_UFFD_WP_ALL                 (MM_CP_UFFD_WP | \
                           MM_CP_UFFD_WP_RESOLVE)
   -bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                 pte_t pte);
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                 pte_t pte, int max_len, int *len);
+#define can_change_pte_writable(vma, addr, pte)    \
+    can_change_ptes_writable(vma, addr, pte, 1, NULL)

nit: add an extra tab for readability:

#define can_change_pte_writable(vma, addr, pte)    \
        can_change_ptes_writable(vma, addr, pte, 1, NULL)

Quoted 77 lines
include/linux/mm.h +1 -0
+
   extern long change_protection(struct mmu_gather *tlb,
                     struct vm_area_struct *vma, unsigned long start,
                     unsigned long end, unsigned long cp_flags);
mm/mprotect.c +36 -10
index 124612ce3d24..6cd8cdc168fa 100644
@@ -40,25 +40,36 @@
     #include "internal.h"
   -bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-                 pte_t pte)
+bool can_change_ptes_writable(struct vm_area_struct *vma, unsigned long addr,
+                 pte_t pte, int max_len, int *len)
   {
       struct page *page;
+    bool temp_ret;
+    bool ret;
+    int i;
   -    if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-        return false;
+    if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE))) {
+        ret = false;
+        goto out;
+    }
         /* Don't touch entries that are not even readable. */
-    if (pte_protnone(pte))
-        return false;
+    if (pte_protnone(pte)) {
+        ret = false;
+        goto out;
+    }
         /* Do we need write faults for softdirty tracking? */
-    if (pte_needs_soft_dirty_wp(vma, pte))
-        return false;
+    if (pte_needs_soft_dirty_wp(vma, pte)) {
+        ret = false;
+        goto out;
+    }
         /* Do we need write faults for uffd-wp tracking? */
-    if (userfaultfd_pte_wp(vma, pte))
-        return false;
+    if (userfaultfd_pte_wp(vma, pte)) {
+        ret = false;
+        goto out;
+    }
         if (!(vma->vm_flags & VM_SHARED)) {
           /*
@@ -68,7 +79,19 @@ bool can_change_pte_writable(struct vm_area_struct *vma,
 unsigned long addr,
            * any additional checks while holding the PT lock.
            */
           page = vm_normal_page(vma, addr, pte);
-        return page && PageAnon(page) && PageAnonExclusive(page);
+        ret = (page && PageAnon(page) && PageAnonExclusive(page));
+        if (!len)
+            return ret;
+
+        /* Check how many consecutive pages are AnonExclusive or not */
+        for (i = 1; i < max_len; ++i) {
+            ++page;
+            temp_ret = (page && PageAnon(page) && PageAnonExclusive(page));
+            if (temp_ret != ret)
+                break;
+        }
+        *len = i;
+        return ret;
       }
         VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,21 +103,55 @@ bool can_change_pte_writable(struct vm_area_struct *vma,
 unsigned long addr,
        * FS was already notified and we can simply mark the PTE writable
        * just like the write-fault handler would do.
        */
-    return pte_dirty(pte);
+    ret = pte_dirty(pte);
+

mm/mprotect.c bool can_change_pte_writable(struct vm_area_struct *vma, · line 117

Quoted 10 lines
mm/mprotect.c +6 -1
+out:
+    /* The entire batch is guaranteed to have the same return value */
+    if (len)
+        *len = max_len;
+    return ret;
   }
     static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t
 *ptep,
-        pte_t pte, int max_nr_ptes)
+        pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)

We'll almost certainly want more flags here in future. I wonder if it's cleaner just to pass "fpb_t extra_flags" here instead of the bool, then OR them in. You can pass 0 or FPB_IGNORE_SOFT_DIRTY at your 2 callsites. No strong opinion though.

Quoted 84 lines
mm/mprotect.c +32 -3
   {
-    const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+    fpb_t flags = FPB_IGNORE_DIRTY;
   -    if (!folio_test_large(folio) || (max_nr_ptes == 1))
+    if (ignore_soft_dirty)
+        flags |= FPB_IGNORE_SOFT_DIRTY;
+
+    if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
           return 1;
         return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,
                      NULL, NULL, NULL);
   }
   +/**
+ * modify_sub_batch - Identifies a sub-batch which has the same return value
+ * of can_change_pte_writable(), from within a folio batch. max_len is the
+ * max length of the possible sub-batch. sub_batch_idx is the offset from
+ * the start of the original folio batch.
+ */
+static int modify_sub_batch(struct vm_area_struct *vma, struct mmu_gather *tlb,
+        unsigned long addr, pte_t *ptep, pte_t oldpte, pte_t ptent,
+        int max_len, int sub_batch_idx)
+{
+    unsigned long new_addr = addr + sub_batch_idx * PAGE_SIZE;
+    pte_t new_oldpte = pte_advance_pfn(oldpte, sub_batch_idx);
+    pte_t new_ptent = pte_advance_pfn(ptent, sub_batch_idx);
+    pte_t *new_ptep = ptep + sub_batch_idx;
+    int len = 1;
+
+    if (can_change_ptes_writable(vma, new_addr, new_ptent, max_len, &len))
+        new_ptent = pte_mkwrite(new_ptent, vma);
+
+    modify_prot_commit_ptes(vma, new_addr, new_ptep, new_oldpte, new_ptent,
 len);
+    if (pte_needs_flush(new_oldpte, new_ptent))
+        tlb_flush_pte_range(tlb, new_addr, len * PAGE_SIZE);
+    return len;
+}
+
   static long change_pte_range(struct mmu_gather *tlb,
           struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr,
           unsigned long end, pgprot_t newprot, unsigned long cp_flags)
@@ -106,7 +163,7 @@ static long change_pte_range(struct mmu_gather *tlb,
       bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
       bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
       bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
-    int nr_ptes;
+    int sub_batch_idx, max_len, len, nr_ptes;
         tlb_change_page_size(tlb, PAGE_SIZE);
       pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
@@ -121,10 +178,12 @@ static long change_pte_range(struct mmu_gather *tlb,
       flush_tlb_batched_pending(vma->vm_mm);
       arch_enter_lazy_mmu_mode();
       do {
+        sub_batch_idx = 0;
           nr_ptes = 1;
           oldpte = ptep_get(pte);
           if (pte_present(oldpte)) {
               int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
+            struct folio *folio = NULL;
               pte_t ptent;
                 /*
@@ -132,7 +191,6 @@ static long change_pte_range(struct mmu_gather *tlb,
                * pages. See similar comment in change_huge_pmd.
                */
               if (prot_numa) {
-                struct folio *folio;
                   int nid;
                   bool toptier;
   @@ -180,7 +238,8 @@ static long change_pte_range(struct mmu_gather *tlb,

                      toptier) {
  skip_batch:
                      nr_ptes = mprotect_batch(folio, addr, pte,
-                                 oldpte, max_nr_ptes); +                                 oldpte, max_nr_ptes, +                                 true);
                      continue;
                  }
                  if (folio_use_access_time(folio))

mm/mprotect.c +2 -0
@@ -188,6 +247,11 @@ static long change_pte_range(struct mmu_gather *tlb,
                           jiffies_to_msecs(jiffies));
               }
   +            if (!folio)
+                folio = vm_normal_folio(vma, addr, oldpte);
+

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb,

Quoted 10 lines
mm/mprotect.c +2 -2
+            nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+                         max_nr_ptes, false);
               oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
               ptent = pte_modify(oldpte, newprot);
   @@ -209,15 +273,23 @@ static long change_pte_range(struct mmu_gather *tlb,
                * example, if a PTE is already dirty and no other
                * COW or special handling is required.
                */
-            if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-                !pte_write(ptent) &&

Don't you need to keep the !pte_write(ptent) condition here? No need to sub- batch if write is already set.

Quoted 20 lines

-                can_change_pte_writable(vma, addr, ptent)) -                ptent = pte_mkwrite(ptent, vma);

mm/mprotect.c +13 -5
-
-            modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-            if (pte_needs_flush(oldpte, ptent))
-                tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-            pages++;
+            if (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) {
+                max_len = nr_ptes;
+                while (sub_batch_idx < nr_ptes) {
+
+                    /* Get length of sub batch */
+                    len = modify_sub_batch(vma, tlb, addr, pte,
+                                   oldpte, ptent, max_len,
+                                   sub_batch_idx);
+                    sub_batch_idx += len;
+                    max_len -= len;
+                }
+            } else {
+                modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent,
Quoted 4 lines

nr_ptes);
+                if (pte_needs_flush(oldpte, ptent))
+                    tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE); +            }

This if/else block and modify_sub_block() is all pretty ugly. I wonder if something like this would be neater:

            use_sub_batch = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pte_write(ptent) && LIKE_can_change_pte_writable(vma, addr, ptent, folio);

            while (nr_ptes) {
                if (use_sub_batch) sub_nr_ptes = sub_batch(...); else
                    sub_nr_ptes = nr_ptes;

                modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, sub_nr_ptes);
                if (pte_needs_flush(oldpte, ptent)) tlb_flush_pte_range(tlb, addr, sub_nr_ptes * PAGE_SIZE);

                addr += sub_nr_ptes * PAGE_SIZE;
                pte += sub_nr_ptes;
                oldpte = pte_advance_pfn(oldpte, sub_nr_ptes); ptent = pte_advance_pfn(ptent, sub_nr_ptes); nr_ptes -= sub_nr_ptes;
                pages += sub_nr_ptes;
            }

Where:

  - LIKE_can_change_pte_writable() is similar to can_change_pte_writable() but does everything except checking the per-page exclusive flag. Note that we already have the folio so can pass that rather than calling vm_normal_page() again.

Quoted 5 lines

  - sub_batch() can be passed the folio and the index of the first page within the folio and the max number of pages to check (nr_ptes). Then returns the number of pages where exclusive flag is the same.

Obviously they both need better names...

I cannot figure a nice way of implementing your suggestion. We need to get the length
of the sub batch and the true/false return value from can_change_ptes_writable, and we also need to call pte_mkwrite in case the ret is true. Doing that in your suggested
way seems harder to me, along with the fact that it is causing indentation problem by
one extra tab compared to my current implementation.

Quoted 1 line

I don't know, my method looks pretty neat to me : )

How about something like this (compile-tested only)? Personally I find this easier to follow:

---8<---

mm/mprotect.c +72 -15
index 124612ce3d24..97ccc2a92a7b 100644
@@ -40,35 +40,47 @@
 
  #include "internal.h"
 
-bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
-			     pte_t pte)
-{
-	struct page *page;
+enum tristate {
+	TRI_FALSE = 0,
+	TRI_TRUE = 1,
+	TRI_MAYBE = -1,
+};
 
+/*
+ * Returns enum tristate indicating whether the pte can be changed to writable.
+ * If TRI_MAYBE is returned, then the folio is anonymous and the user must
+ * additionally check PageAnonExclusive() for every page in the desired range.
+ */
+static int maybe_change_pte_writable(struct vm_area_struct *vma,
+				     unsigned long addr, pte_t pte,
+				     struct folio *folio)
+{
  	if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE)))
-		return false;
+		return TRI_FALSE;
 
  	/* Don't touch entries that are not even readable. */
  	if (pte_protnone(pte))
-		return false;
+		return TRI_FALSE;
 
  	/* Do we need write faults for softdirty tracking? */
  	if (pte_needs_soft_dirty_wp(vma, pte))
-		return false;
+		return TRI_FALSE;
 
  	/* Do we need write faults for uffd-wp tracking? */
  	if (userfaultfd_pte_wp(vma, pte))
-		return false;
+		return TRI_FALSE;
 
  	if (!(vma->vm_flags & VM_SHARED)) {
  		/*
  		 * Writable MAP_PRIVATE mapping: We can only special-case on
  		 * exclusive anonymous pages, because we know that our
  		 * write-fault handler similarly would map them writable without
-		 * any additional checks while holding the PT lock.
+		 * any additional checks while holding the PT lock. So if the
+		 * folio is not anonymous, we know we cannot change pte
+		 * writable. If it is anonymous then the caller must further
+		 * check that the page is AnonExclusive().
  		 */
-		page = vm_normal_page(vma, addr, pte);
-		return page && PageAnon(page) && PageAnonExclusive(page);
+		return (!folio || folio_test_anon(folio)) ? TRI_MAYBE : TRI_FALSE;
  	}
 
  	VM_WARN_ON_ONCE(is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte));
@@ -80,15 +92,60 @@ bool can_change_pte_writable(struct vm_area_struct *vma,
 unsigned long addr,
  	 * FS was already notified and we can simply mark the PTE writable
  	 * just like the write-fault handler would do.
  	 */
-	return pte_dirty(pte);
+	return pte_dirty(pte) ? TRI_TRUE : TRI_FALSE;
+}
+
+/*
+ * Returns the number of pages within the folio, starting from the page
+ * indicated by pgidx and up to pgidx + max_nr, that have the same value of
+ * PageAnonExclusive(). Must only be called for anonymous folios. Value of
+ * PageAnonExclusive() is returned in *exclusive.
+ */
+static int anon_exclusive_batch(struct folio *folio, int pgidx, int max_nr,
+				bool *exclusive)
+{
+	VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio);
+	VM_WARN_ON_FOLIO(folio_nr_pages(folio) < pgidx + max_nr, folio);
+
+	struct page *page = folio_page(folio, pgidx++);
+	int nr = 1;
+
+	*exclusive = PageAnonExclusive(page);
+
+	while (nr < max_nr) {
+		page = folio_page(folio, pgidx++);
+		if (*exclusive != PageAnonExclusive(page))
+			break;
+		nr++;
+	}
+
+	return nr;
+}
+
+bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t pte)
+{
+	int ret;
+	struct page *page;
+
+	ret = maybe_change_pte_writable(vma, addr, pte, NULL);
+	if (ret == TRI_MAYBE) {
+		page = vm_normal_page(vma, addr, pte);
+		ret = page && PageAnon(page) && PageAnonExclusive(page);
+	}
+
+	return ret;
  }
 
  static int mprotect_batch(struct folio *folio, unsigned long addr, pte_t *ptep,
-		pte_t pte, int max_nr_ptes)
+		pte_t pte, int max_nr_ptes, bool ignore_soft_dirty)
  {
-	const fpb_t flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	fpb_t flags = FPB_IGNORE_DIRTY;
 
-	if (!folio_test_large(folio) || (max_nr_ptes == 1))
+	if (ignore_soft_dirty)
+		flags |= FPB_IGNORE_SOFT_DIRTY;
+
+	if (!folio || !folio_test_large(folio) || (max_nr_ptes == 1))
  		return 1;
 

return folio_pte_batch(folio, addr, ptep, pte, max_nr_ptes, flags,

mm/mprotect.c +38 -12
@@ -125,14 +182,17 @@ static long change_pte_range(struct mmu_gather *tlb,
  		oldpte = ptep_get(pte);
  		if (pte_present(oldpte)) {
  			int max_nr_ptes = (end - addr) >> PAGE_SHIFT;
-			pte_t ptent;
+			struct folio *folio = NULL;
+			int sub_nr_ptes, pgidx;
+			pte_t ptent, newpte;
+			bool sub_set_write;
+			int set_write;
 
  			/*
  			 * Avoid trapping faults against the zero or KSM
  			 * pages. See similar comment in change_huge_pmd.
  			 */
  			if (prot_numa) {
-				struct folio *folio;
  				int nid;
  				bool toptier;
 
@@ -180,7 +240,8 @@ static long change_pte_range(struct mmu_gather *tlb,
  				    toptier) {
  skip_batch:
  					nr_ptes = mprotect_batch(folio, addr, pte,
-								 oldpte, max_nr_ptes);
+								 oldpte, max_nr_ptes,
+								 true);
  					continue;
  				}
  				if (folio_use_access_time(folio))
@@ -188,6 +249,11 @@ static long change_pte_range(struct mmu_gather *tlb,
  						jiffies_to_msecs(jiffies));
  			}
 
+			if (!folio)
+				folio = vm_normal_folio(vma, addr, oldpte);
+
+			nr_ptes = mprotect_batch(folio, addr, pte, oldpte,
+						 max_nr_ptes, false);
  			oldpte = modify_prot_start_ptes(vma, addr, pte, nr_ptes);
  			ptent = pte_modify(oldpte, newprot);
 
@@ -209,15 +275,38 @@ static long change_pte_range(struct mmu_gather *tlb,
  			 * example, if a PTE is already dirty and no other
  			 * COW or special handling is required.
  			 */
-			if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
-			    !pte_write(ptent) &&
-			    can_change_pte_writable(vma, addr, ptent))
-				ptent = pte_mkwrite(ptent, vma);
-
-			modify_prot_commit_ptes(vma, addr, pte, oldpte, ptent, nr_ptes);
-			if (pte_needs_flush(oldpte, ptent))
-				tlb_flush_pte_range(tlb, addr, PAGE_SIZE);
-			pages++;
+			set_write = (cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
+				    !pte_write(ptent) &&
+				    maybe_change_pte_writable(vma, addr, ptent, folio);
+
+			while (nr_ptes) {
+				if (set_write == TRI_MAYBE) {
+					pgidx = folio_pfn(folio) - pte_pfn(ptent);
+					sub_nr_ptes = anon_exclusive_batch(folio,
+						pgidx, nr_ptes, &sub_set_write);
+				} else {
+					sub_nr_ptes = nr_ptes;
+					sub_set_write = set_write == TRI_TRUE;
+				}
+
+				if (sub_set_write)
+					newpte = pte_mkwrite(ptent, vma);
+				else
+					newpte = ptent;
+
+				modify_prot_commit_ptes(vma, addr, pte, oldpte,
+							newpte, sub_nr_ptes);
+				if (pte_needs_flush(oldpte, newpte))
+					tlb_flush_pte_range(tlb, addr,
+						sub_nr_ptes * PAGE_SIZE);
+
+				addr += sub_nr_ptes * PAGE_SIZE;

mm/mprotect.c static long change_pte_range(struct mmu_gather *tlb,

Quoted 10 lines
mm/mprotect.c +6 -1
+				pte += sub_nr_ptes;
+				oldpte = pte_advance_pfn(oldpte, sub_nr_ptes);
+				ptent = pte_advance_pfn(ptent, sub_nr_ptes);
+				nr_ptes -= sub_nr_ptes;
+				pages += sub_nr_ptes;
+			}
  		} else if (is_swap_pte(oldpte)) {
  			swp_entry_t entry = pte_to_swp_entry(oldpte);
  			pte_t newpte;
---8<---

Looks quite clean! I'll try this.

Quoted 3 lines

Thanks,
Ryan

author
#28 lore raw

PATCH v3 4/5 arm64: Add batched version of ptep_modify_prot_start

In reply to Dev Jain

Override the generic definition to use get_and_clear_full_ptes(). This helper does a TLBI only for the starting and ending contpte block of the range, whereas the current implementation will call ptep_get_and_clear() for every contpte block, thus doing a TLBI on every contpte block. Therefore, we have a performance win. The arm64 definition of pte_accessible() allows us to batch around it in clear_flush_ptes():
#define pte_accessible(mm, pte) \
(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))

All ptes are obviously present in the folio batch, and they are also valid.

---
 arch/arm64/include/asm/pgtable.h |  5 +++++
 arch/arm64/mm/mmu.c              | 12 +++++++++---
 include/linux/pgtable.h          |  4 ++++
 mm/pgtable-generic.c             | 16 +++++++++++-----
 4 files changed, 29 insertions(+), 8 deletions(-)
arch/arm64/include/asm/pgtable.h +5 -0
index 2a77f11b78d5..8872ea5f0642 100644
@@ -1553,6 +1553,11 @@ extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
 				    unsigned long addr, pte_t *ptep,
 				    pte_t old_pte, pte_t new_pte);
 
+#define modify_prot_start_ptes modify_prot_start_ptes
+extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
+				    unsigned long addr, pte_t *ptep,
+				    unsigned int nr);
+
 #ifdef CONFIG_ARM64_CONTPTE
 
 /*
arch/arm64/mm/mmu.c +9 -3
index 8fcf59ba39db..fe60be8774f4 100644
@@ -1523,7 +1523,8 @@ static int __init prevent_bootmem_remove_init(void)
 early_initcall(prevent_bootmem_remove_init);
 #endif
 
-pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t *ptep, unsigned int nr)
 {
 	if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
 		/*
@@ -1532,9 +1533,14 @@ pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte
 		 * in cases where cpu is affected with errata #2645198.
 		 */
 		if (pte_user_exec(ptep_get(ptep)))
-			return ptep_clear_flush(vma, addr, ptep);
+			return clear_flush_ptes(vma, addr, ptep, nr);
 	}
-	return ptep_get_and_clear(vma->vm_mm, addr, ptep);
+	return get_and_clear_full_ptes(vma->vm_mm, addr, ptep, nr, 0);
+}
+
+pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+{
+	return modify_prot_start_ptes(vma, addr, ptep, 1);
 }
 
 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
include/linux/pgtable.h +4 -0
index e40ed57e034d..41f4a8de5c28 100644
@@ -828,6 +828,10 @@ extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
 			      pte_t *ptep);
 #endif
 
+extern pte_t clear_flush_ptes(struct vm_area_struct *vma,
+			      unsigned long address,
+			      pte_t *ptep, unsigned int nr);
+
 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
 			      unsigned long address,
mm/pgtable-generic.c +11 -5
index 5a882f2b10f9..e238f88c3cac 100644
@@ -90,17 +90,23 @@ int ptep_clear_flush_young(struct vm_area_struct *vma,
 }
 #endif
 
-#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
-pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
-		       pte_t *ptep)
+pte_t clear_flush_ptes(struct vm_area_struct *vma, unsigned long address,
+		       pte_t *ptep, unsigned int nr)
 {
 	struct mm_struct *mm = (vma)->vm_mm;
 	pte_t pte;
-	pte = ptep_get_and_clear(mm, address, ptep);
+	pte = get_and_clear_full_ptes(mm, address, ptep, nr, 0);
 	if (pte_accessible(mm, pte))
-		flush_tlb_page(vma, address);
+		flush_tlb_range(vma, address, address + nr * PAGE_SIZE);
 	return pte;
 }
+
+#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
+pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
+		       pte_t *ptep)
+{
+	return clear_flush_ptes(vma, address, ptep, 1);
+}
 #endif
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
Signature · 1 line
2.30.2
#29 lore raw

Re: [PATCH v3 4/5] arm64: Add batched version of ptep_modify_prot_start

Quoted 11 lines · Dev Jain

Override the generic definition to use get_and_clear_full_ptes(). This helper does a TLBI only for the starting and ending contpte block of the range, whereas the current implementation will call ptep_get_and_clear() for every contpte block, thus doing a TLBI on every contpte block. Therefore, we have a performance win. The arm64 definition of pte_accessible() allows us to batch around it in clear_flush_ptes():
#define pte_accessible(mm, pte) \
(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))

All ptes are obviously present in the folio batch, and they are also valid.

Please squash this with the

Quoted 38 lines
---
 arch/arm64/include/asm/pgtable.h |  5 +++++
 arch/arm64/mm/mmu.c              | 12 +++++++++---
 include/linux/pgtable.h          |  4 ++++
 mm/pgtable-generic.c             | 16 +++++++++++-----
 4 files changed, 29 insertions(+), 8 deletions(-)
arch/arm64/include/asm/pgtable.h +5 -0
index 2a77f11b78d5..8872ea5f0642 100644
@@ -1553,6 +1553,11 @@ extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
 				    unsigned long addr, pte_t *ptep,
 				    pte_t old_pte, pte_t new_pte);
 
+#define modify_prot_start_ptes modify_prot_start_ptes
+extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
+				    unsigned long addr, pte_t *ptep,
+				    unsigned int nr);
+
 #ifdef CONFIG_ARM64_CONTPTE
 
 /*
arch/arm64/mm/mmu.c +2 -1
index 8fcf59ba39db..fe60be8774f4 100644
@@ -1523,7 +1523,8 @@ static int __init prevent_bootmem_remove_init(void)
 early_initcall(prevent_bootmem_remove_init);
 #endif
 
-pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t *ptep, unsigned int nr)
 {
 	if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
 		/*

arch/arm64/mm/mmu.c pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte · line 1539

Quoted 9 lines
arch/arm64/mm/mmu.c +3 -2
@@ -1532,9 +1533,14 @@ pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte
 		 * in cases where cpu is affected with errata #2645198.
 		 */
 		if (pte_user_exec(ptep_get(ptep)))
-			return ptep_clear_flush(vma, addr, ptep);
+			return clear_flush_ptes(vma, addr, ptep, nr);
 	}
-	return ptep_get_and_clear(vma->vm_mm, addr, ptep);
+	return get_and_clear_full_ptes(vma->vm_mm, addr, ptep, nr, 0);
+}

I think we can do this more precisely with respect to tlbis and also without needing to create a new clear_flush_ptes() helper:

pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
pte_t *ptep, unsigned int nr)

{
	pte_t pte = get_and_clear_full_ptes(vma->vm_mm, addr, ptep, nr, 0);

	if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
		/*
		 * Break-before-make (BBM) is required for all user space mappings
		 * when the permission changes from executable to non-executable
		 * in cases where cpu is affected with errata #2645198.
		 */
		if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte))
			__flush_tlb_range(vma, addr, nr * PAGE_SIZE,
					  PAGE_SIZE, true, 3);
	}

	return pte;
}
Quoted 35 lines
arch/arm64/mm/mmu.c +4 -0
+
+pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+{
+	return modify_prot_start_ptes(vma, addr, ptep, 1);
 }
 
 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
include/linux/pgtable.h +4 -0
index e40ed57e034d..41f4a8de5c28 100644
@@ -828,6 +828,10 @@ extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
 			      pte_t *ptep);
 #endif
 
+extern pte_t clear_flush_ptes(struct vm_area_struct *vma,
+			      unsigned long address,
+			      pte_t *ptep, unsigned int nr);
+
 #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
 extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
 			      unsigned long address,
mm/pgtable-generic.c +2 -3
index 5a882f2b10f9..e238f88c3cac 100644
@@ -90,17 +90,23 @@ int ptep_clear_flush_young(struct vm_area_struct *vma,
 }
 #endif
 
-#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
-pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
-		       pte_t *ptep)
+pte_t clear_flush_ptes(struct vm_area_struct *vma, unsigned long address,
+		       pte_t *ptep, unsigned int nr)

mm/pgtable-generic.c int ptep_clear_flush_young(struct vm_area_struct *vma, · line 102

Quoted 10 lines
mm/pgtable-generic.c +2 -2
 {
 	struct mm_struct *mm = (vma)->vm_mm;
 	pte_t pte;
-	pte = ptep_get_and_clear(mm, address, ptep);
+	pte = get_and_clear_full_ptes(mm, address, ptep, nr, 0);
 	if (pte_accessible(mm, pte))
-		flush_tlb_page(vma, address);
+		flush_tlb_range(vma, address, address + nr * PAGE_SIZE);
 	return pte;
 }

Let's not create a new generic helper if only arm64 is using it. We would also want to add a doc header to describe this helper. My proposal avoids this.

Thanks,
Ryan

Quoted 10 lines
mm/pgtable-generic.c +7 -0
+
+#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
+pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
+		       pte_t *ptep)
+{
+	return clear_flush_ptes(vma, address, ptep, 1);
+}
 #endif
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
author
#30 lore raw

Re: [PATCH v3 4/5] arm64: Add batched version of ptep_modify_prot_start

Quoted 125 lines · Ryan Roberts
Quoted 11 lines · Dev Jain

Override the generic definition to use get_and_clear_full_ptes(). This helper does a TLBI only for the starting and ending contpte block of the range, whereas the current implementation will call ptep_get_and_clear() for every contpte block, thus doing a TLBI on every contpte block. Therefore, we have a performance win. The arm64 definition of pte_accessible() allows us to batch around it in clear_flush_ptes():
#define pte_accessible(mm, pte) \
(mm_tlb_flush_pending(mm) ? pte_present(pte) : pte_valid(pte))

All ptes are obviously present in the folio batch, and they are also valid.

Please squash this with the

Quoted 38 lines
---
  arch/arm64/include/asm/pgtable.h |  5 +++++
  arch/arm64/mm/mmu.c              | 12 +++++++++---
  include/linux/pgtable.h          |  4 ++++
  mm/pgtable-generic.c             | 16 +++++++++++-----
  4 files changed, 29 insertions(+), 8 deletions(-)
arch/arm64/include/asm/pgtable.h +5 -0
index 2a77f11b78d5..8872ea5f0642 100644
@@ -1553,6 +1553,11 @@ extern void ptep_modify_prot_commit(struct vm_area_struct *vma,
  				    unsigned long addr, pte_t *ptep,
  				    pte_t old_pte, pte_t new_pte);
 
+#define modify_prot_start_ptes modify_prot_start_ptes
+extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
+				    unsigned long addr, pte_t *ptep,
+				    unsigned int nr);
+
  #ifdef CONFIG_ARM64_CONTPTE
 
  /*
arch/arm64/mm/mmu.c +2 -1
index 8fcf59ba39db..fe60be8774f4 100644
@@ -1523,7 +1523,8 @@ static int __init prevent_bootmem_remove_init(void)
  early_initcall(prevent_bootmem_remove_init);
  #endif
 
-pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t *ptep, unsigned int nr)
  {
  	if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
  		/*

arch/arm64/mm/mmu.c pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte · line 1539

Quoted 9 lines
arch/arm64/mm/mmu.c +3 -2
@@ -1532,9 +1533,14 @@ pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte
  		 * in cases where cpu is affected with errata #2645198.
  		 */
  		if (pte_user_exec(ptep_get(ptep)))
-			return ptep_clear_flush(vma, addr, ptep);
+			return clear_flush_ptes(vma, addr, ptep, nr);
  	}
-	return ptep_get_and_clear(vma->vm_mm, addr, ptep);
+	return get_and_clear_full_ptes(vma->vm_mm, addr, ptep, nr, 0);
+}

I think we can do this more precisely with respect to tlbis and also without needing to create a new clear_flush_ptes() helper:

pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr,
pte_t *ptep, unsigned int nr)

{
	pte_t pte = get_and_clear_full_ptes(vma->vm_mm, addr, ptep, nr, 0);

	if (alternative_has_cap_unlikely(ARM64_WORKAROUND_2645198)) {
		/*
		 * Break-before-make (BBM) is required for all user space mappings
		 * when the permission changes from executable to non-executable
		 * in cases where cpu is affected with errata #2645198.
		 */
		if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte))
			__flush_tlb_range(vma, addr, nr * PAGE_SIZE,
					  PAGE_SIZE, true, 3);
	}

	return pte;
}
Quoted 40 lines
arch/arm64/mm/mmu.c +4 -0
+
+pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+{
+	return modify_prot_start_ptes(vma, addr, ptep, 1);
  }
 
  void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
include/linux/pgtable.h +4 -0
index e40ed57e034d..41f4a8de5c28 100644
@@ -828,6 +828,10 @@ extern pte_t ptep_clear_flush(struct vm_area_struct *vma,
  			      pte_t *ptep);
  #endif
 
+extern pte_t clear_flush_ptes(struct vm_area_struct *vma,
+			      unsigned long address,
+			      pte_t *ptep, unsigned int nr);
+
  #ifndef __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH
  extern pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma,
  			      unsigned long address,
mm/pgtable-generic.c +3 -4
index 5a882f2b10f9..e238f88c3cac 100644
@@ -90,17 +90,23 @@ int ptep_clear_flush_young(struct vm_area_struct *vma,
  }
  #endif
 
-#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
-pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
-		       pte_t *ptep)
+pte_t clear_flush_ptes(struct vm_area_struct *vma, unsigned long address,
+		       pte_t *ptep, unsigned int nr)
  {
  	struct mm_struct *mm = (vma)->vm_mm;
  	pte_t pte;
-	pte = ptep_get_and_clear(mm, address, ptep);
+	pte = get_and_clear_full_ptes(mm, address, ptep, nr, 0);
Quoted 8 lines

mm/pgtable-generic.c int ptep_clear_flush_young(struct vm_area_struct *vma, · line 102

Quoted 5 lines
mm/pgtable-generic.c +1 -1
  	if (pte_accessible(mm, pte))
-		flush_tlb_page(vma, address);
+		flush_tlb_range(vma, address, address + nr * PAGE_SIZE);
  	return pte;
  }

Let's not create a new generic helper if only arm64 is using it. We would also want to add a doc header to describe this helper. My proposal avoids this.

This is better, thanks!

Quoted 15 lines

Thanks,
Ryan

Quoted 10 lines
mm/pgtable-generic.c +7 -0
+
+#ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH
+pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address,
+		       pte_t *ptep)
+{
+	return clear_flush_ptes(vma, address, ptep, 1);
+}
  #endif
 
  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
author
#31 lore raw

PATCH v3 5/5 arm64: Add batched version of ptep_modify_prot_commit

In reply to Dev Jain

Override the generic definition to simply use set_ptes() to map the new ptes into the pagetable.

---
 arch/arm64/include/asm/pgtable.h | 5 +++++
 arch/arm64/mm/mmu.c              | 9 ++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)
arch/arm64/include/asm/pgtable.h +5 -0
index 8872ea5f0642..0b13ca38f80c 100644
@@ -1558,6 +1558,11 @@ extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
 				    unsigned long addr, pte_t *ptep,
 				    unsigned int nr);
 
+#define modify_prot_commit_ptes modify_prot_commit_ptes
+extern void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+				    pte_t *ptep, pte_t old_pte, pte_t pte,
+				    unsigned int nr);
+
 #ifdef CONFIG_ARM64_CONTPTE
 
 /*
arch/arm64/mm/mmu.c +8 -1
index fe60be8774f4..5f04bcdcd946 100644
@@ -1543,10 +1543,17 @@ pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte
 	return modify_prot_start_ptes(vma, addr, ptep, 1);
 }
 
+void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t *ptep, pte_t old_pte, pte_t pte,
+			     unsigned int nr)
+{
+	set_ptes(vma->vm_mm, addr, ptep, pte, nr);
+}
+
 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
 			     pte_t old_pte, pte_t pte)
 {
-	set_pte_at(vma->vm_mm, addr, ptep, pte);
+	modify_prot_commit_ptes(vma, addr, ptep, old_pte, pte, 1);
 }
 
 /*
Signature · 1 line
2.30.2
#32 lore raw

Re: [PATCH v3 5/5] arm64: Add batched version of ptep_modify_prot_commit

Quoted 4 lines · Dev Jain

Override the generic definition to simply use set_ptes() to map the new ptes into the pagetable.

Let's squash this into the previous patch; it doesn't make sense for an arch to implement one without the other. Otherwise, LGTM.

Thanks,
Ryan

Quoted 44 lines
---
 arch/arm64/include/asm/pgtable.h | 5 +++++
 arch/arm64/mm/mmu.c              | 9 ++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)
arch/arm64/include/asm/pgtable.h +5 -0
index 8872ea5f0642..0b13ca38f80c 100644
@@ -1558,6 +1558,11 @@ extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
 				    unsigned long addr, pte_t *ptep,
 				    unsigned int nr);
 
+#define modify_prot_commit_ptes modify_prot_commit_ptes
+extern void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+				    pte_t *ptep, pte_t old_pte, pte_t pte,
+				    unsigned int nr);
+
 #ifdef CONFIG_ARM64_CONTPTE
 
 /*
arch/arm64/mm/mmu.c +8 -1
index fe60be8774f4..5f04bcdcd946 100644
@@ -1543,10 +1543,17 @@ pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte
 	return modify_prot_start_ptes(vma, addr, ptep, 1);
 }
 
+void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t *ptep, pte_t old_pte, pte_t pte,
+			     unsigned int nr)
+{
+	set_ptes(vma->vm_mm, addr, ptep, pte, nr);
+}
+
 void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
 			     pte_t old_pte, pte_t pte)
 {
-	set_pte_at(vma->vm_mm, addr, ptep, pte);
+	modify_prot_commit_ptes(vma, addr, ptep, old_pte, pte, 1);
 }
 
 /*
author
#33 lore raw

Re: [PATCH v3 5/5] arm64: Add batched version of ptep_modify_prot_commit

Quoted 7 lines · Ryan Roberts
Quoted 4 lines · Dev Jain

Override the generic definition to simply use set_ptes() to map the new ptes into the pagetable.

Let's squash this into the previous patch; it doesn't make sense for an arch to implement one without the other. Otherwise, LGTM.

Sure. Thanks.

Quoted 49 lines

Thanks,
Ryan

Quoted 44 lines
---
  arch/arm64/include/asm/pgtable.h | 5 +++++
  arch/arm64/mm/mmu.c              | 9 ++++++++-
  2 files changed, 13 insertions(+), 1 deletion(-)
arch/arm64/include/asm/pgtable.h +5 -0
index 8872ea5f0642..0b13ca38f80c 100644
@@ -1558,6 +1558,11 @@ extern pte_t modify_prot_start_ptes(struct vm_area_struct *vma,
  				    unsigned long addr, pte_t *ptep,
  				    unsigned int nr);
 
+#define modify_prot_commit_ptes modify_prot_commit_ptes
+extern void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+				    pte_t *ptep, pte_t old_pte, pte_t pte,
+				    unsigned int nr);
+
  #ifdef CONFIG_ARM64_CONTPTE
 
  /*
arch/arm64/mm/mmu.c +8 -1
index fe60be8774f4..5f04bcdcd946 100644
@@ -1543,10 +1543,17 @@ pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr, pte
  	return modify_prot_start_ptes(vma, addr, ptep, 1);
  }
 
+void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned long addr,
+			     pte_t *ptep, pte_t old_pte, pte_t pte,
+			     unsigned int nr)
+{
+	set_ptes(vma->vm_mm, addr, ptep, pte, nr);
+}
+
  void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
  			     pte_t old_pte, pte_t pte)
  {
-	set_pte_at(vma->vm_mm, addr, ptep, pte);
+	modify_prot_commit_ptes(vma, addr, ptep, old_pte, pte, 1);
  }
 
  /*