Skip to contentSkip to conversation

mempolicy: Optimize queue_folios_pte_range by PTE batching

PATCH Superseded Dev Jain 2 participants 3 messages Apr 15, 2025 – Apr 16, 2025

Subsystems (2)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Dev Jain

author
#0 lore raw

PATCH v2 1/1 mempolicy: Optimize queue_folios_pte_range by PTE batching

After the check for queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.

With a test program migrating pages of the calling process, which includes a mapped VMA of size 4GB with pte-mapped large folios of order-9, and migrating once back and forth node-0 and node-1, the average execution time reduces from 7.5 to 4 seconds, giving an approx 47% speedup.

v1->v2:
- Follow reverse xmas tree declarations
- Don't initialize nr
- Move folio_pte_batch() immediately after retrieving a normal folio
- increment nr_failed in one shot

---
 mm/mempolicy.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
mm/mempolicy.c +10 -2
index b28a1e6ae096..ca90cdcd3207 100644
@@ -566,6 +566,7 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
 static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
 			unsigned long end, struct mm_walk *walk)
 {
+	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
 	struct vm_area_struct *vma = walk->vma;
 	struct folio *folio;
 	struct queue_pages *qp = walk->private;
@@ -573,6 +574,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
 	pte_t *pte, *mapped_pte;
 	pte_t ptent;
 	spinlock_t *ptl;
+	int max_nr, nr;
 
 	ptl = pmd_trans_huge_lock(pmd, vma);
 	if (ptl) {
@@ -586,7 +588,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
 		walk->action = ACTION_AGAIN;
 		return 0;
 	}
-	for (; addr != end; pte++, addr += PAGE_SIZE) {
+	for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) {
+		nr = 1;
 		ptent = ptep_get(pte);
 		if (pte_none(ptent))
 			continue;
@@ -598,6 +601,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
 		folio = vm_normal_folio(vma, addr, ptent);
 		if (!folio || folio_is_zone_device(folio))
 			continue;
+		if (folio_test_large(folio) &&
+		   (max_nr = ((end - addr) >> PAGE_SHIFT)) != 1)
+			nr = folio_pte_batch(folio, addr, pte, ptent,
+					     max_nr, fpb_flags,
+					     NULL, NULL, NULL);
 		/*
 		 * vm_normal_folio() filters out zero pages, but there might
 		 * still be reserved folios to skip, perhaps in a VDSO.
@@ -630,7 +638,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
 		if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) ||
 		    !vma_migratable(vma) ||
 		    !migrate_folio_add(folio, qp->pagelist, flags)) {
-			qp->nr_failed++;
+			qp->nr_failed += nr;
 			if (strictly_unmovable(flags))
 				break;
 		}
Signature · 1 line
2.30.2
Acked-by on v2 1/1
#1 lore raw
Quoted 51 lines · Dev Jain

After the check for queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.

With a test program migrating pages of the calling process, which includes a mapped VMA of size 4GB with pte-mapped large folios of order-9, and migrating once back and forth node-0 and node-1, the average execution time reduces from 7.5 to 4 seconds, giving an approx 47% speedup.

v1->v2:
- Follow reverse xmas tree declarations
- Don't initialize nr
- Move folio_pte_batch() immediately after retrieving a normal folio
- increment nr_failed in one shot

---
  mm/mempolicy.c | 12 ++++++++++--
  1 file changed, 10 insertions(+), 2 deletions(-)
mm/mempolicy.c +4 -1
index b28a1e6ae096..ca90cdcd3207 100644
@@ -566,6 +566,7 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk)
  static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
  			unsigned long end, struct mm_walk *walk)
  {
+	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
  	struct vm_area_struct *vma = walk->vma;
  	struct folio *folio;
  	struct queue_pages *qp = walk->private;
@@ -573,6 +574,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
  	pte_t *pte, *mapped_pte;
  	pte_t ptent;
  	spinlock_t *ptl;
+	int max_nr, nr;
 
  	ptl = pmd_trans_huge_lock(pmd, vma);
  	if (ptl) {
@@ -586,7 +588,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
  		walk->action = ACTION_AGAIN;
  		return 0;
  	}
-	for (; addr != end; pte++, addr += PAGE_SIZE) {
+	for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) {
+		nr = 1;
  		ptent = ptep_get(pte);
  		if (pte_none(ptent))
  			continue;

mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, · line 605

Quoted 5 lines
mm/mempolicy.c +2 -0
@@ -598,6 +601,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
  		folio = vm_normal_folio(vma, addr, ptent);
  		if (!folio || folio_is_zone_device(folio))
  			continue;
+		if (folio_test_large(folio) &&
+		   (max_nr = ((end - addr) >> PAGE_SHIFT)) != 1)

That's real nasty :)

Let's simply do at the beginning of the loop:

	max_nr = (end - addr) >> PAGE_SHIFT;
	nr = 1;

Then here

if (folio_test_large(folio) && max_nr != 1)

	nr = ...

The compiler is smart enough to optimize the computation of values where really required.

With that

Thanks!

Signature · 3 lines
Cheers,

David / dhildenb
author
#2 lore raw
Quoted 68 lines · David Hildenbrand
Quoted 53 lines · Dev Jain

After the check for queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.

With a test program migrating pages of the calling process, which includes a mapped VMA of size 4GB with pte-mapped large folios of order-9, and migrating once back and forth node-0 and node-1, the average execution time reduces from 7.5 to 4 seconds, giving an approx 47% speedup.

v1->v2:
  - Follow reverse xmas tree declarations
  - Don't initialize nr
  - Move folio_pte_batch() immediately after retrieving a normal folio
  - increment nr_failed in one shot

---
  mm/mempolicy.c | 12 ++++++++++--

  1 file changed, 10 insertions(+), 2 deletions(-)
mm/mempolicy.c +1 -0
index b28a1e6ae096..ca90cdcd3207 100644
@@ -566,6 +566,7 @@ static void queue_folios_pmd(pmd_t *pmd, struct 
 mm_walk *walk)
   static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,
               unsigned long end, struct mm_walk *walk)
   {
+    const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
       struct vm_area_struct *vma = walk->vma;
       struct folio *folio;

      struct queue_pages *qp = walk->private;

mm/mempolicy.c +3 -1
@@ -573,6 +574,7 @@ static int queue_folios_pte_range(pmd_t *pmd, 
 unsigned long addr,
       pte_t *pte, *mapped_pte;
       pte_t ptent;
       spinlock_t *ptl;
+    int max_nr, nr;
       ptl = pmd_trans_huge_lock(pmd, vma);
       if (ptl) {
@@ -586,7 +588,8 @@ static int queue_folios_pte_range(pmd_t *pmd, 
 unsigned long addr,
           walk->action = ACTION_AGAIN;
           return 0;
       }
-    for (; addr != end; pte++, addr += PAGE_SIZE) {
+    for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) {
+        nr = 1;
           ptent = ptep_get(pte);
           if (pte_none(ptent))

              continue;

mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 606

Quoted 6 lines
mm/mempolicy.c +2 -0
@@ -598,6 +601,11 @@ static int queue_folios_pte_range(pmd_t *pmd, 
 unsigned long addr,
           folio = vm_normal_folio(vma, addr, ptent);
           if (!folio || folio_is_zone_device(folio))
               continue;
+        if (folio_test_large(folio) &&
+           (max_nr = ((end - addr) >> PAGE_SHIFT)) != 1)

That's real nasty :)

Let's simply do at the beginning of the loop:

    max_nr = (end - addr) >> PAGE_SHIFT;
    nr = 1;

Quoted 6 lines

Then here

if (folio_test_large(folio) && max_nr != 1)
    nr = ...

The compiler is smart enough to optimize the computation of values where really required.

If that's the case, I'll change it, thanks.

Quoted 4 lines

With that

Thanks!

Quoted 3 lines

Thanks!