Skip to contentSkip to conversation

mm/mlock: implement folio_mlock_step() using folio_pte_batch()

PATCH Superseded Lance Yang 5 participants 7 messages Jun 3, 2024

Subsystems (2)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Lance Yang

author
#0 lore raw

PATCH 1/1 mm/mlock: implement folio_mlock_step() using folio_pte_batch()

Let's make folio_mlock_step() simply a wrapper around folio_pte_batch(), which will greatly reduce the cost of ptep_get() when scanning a range of contptes.

---
 mm/mlock.c | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)
mm/mlock.c +6 -17
index 30b51cdea89d..1ae6232d38cf 100644
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
 static inline unsigned int folio_mlock_step(struct folio *folio,
 		pte_t *pte, unsigned long addr, unsigned long end)
 {
-	unsigned int count, i, nr = folio_nr_pages(folio);
-	unsigned long pfn = folio_pfn(folio);
-	pte_t ptent = ptep_get(pte);
-
-	if (!folio_test_large(folio))
+	if (likely(!folio_test_large(folio)))
 		return 1;
 
-	count = pfn + nr - pte_pfn(ptent);
-	count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-	for (i = 0; i < count; i++, pte++) {
-		pte_t entry = ptep_get(pte);
-
-		if (!pte_present(entry))
-			break;
-		if (pte_pfn(entry) - pfn >= nr)
-			break;
-	}
+	const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+	int max_nr = (end - addr) / PAGE_SIZE;
+	pte_t ptent = ptep_get(pte);
 
-	return i;
+	return folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, NULL,
+			       NULL, NULL);
 }
 
 static inline bool allow_mlock_munlock(struct folio *folio,
Signature · 1 line
2.33.1
#1 lore raw

mm/mlock.c void munlock_folio(struct folio *folio) · line 312

Quoted 4 lines · Lance Yang

{

mm/mlock.c +0 -3
-	unsigned int count, i, nr = folio_nr_pages(folio);
-	unsigned long pfn = folio_pfn(folio);
-	pte_t ptent = ptep_get(pte);

Please don't move type declarations later in the function. Just because you can doesn't mean you should.

mm/mlock.c void munlock_folio(struct folio *folio) · line 311

Quoted 3 lines
mm/mlock.c +1 -1
-	if (!folio_test_large(folio))
+	if (likely(!folio_test_large(folio)))
 		return 1;

How likely is this now? How likely will it be in two years time? Does this actually make any difference in either code generation or performance?

author
#2 lore raw

Hi Matthew,

Thanks for taking time to review!

Quoted 9 lines · Matthew Wilcox

mm/mlock.c void munlock_folio(struct folio *folio) · line 312

Quoted 4 lines · Lance Yang

{

mm/mlock.c +0 -3
-     unsigned int count, i, nr = folio_nr_pages(folio);
-     unsigned long pfn = folio_pfn(folio);
-     pte_t ptent = ptep_get(pte);

Please don't move type declarations later in the function. Just because you can doesn't mean you should.

Thanks for pointing this out, I'll adjust as you suggested.

Quoted 8 lines

mm/mlock.c void munlock_folio(struct folio *folio) · line 311

Quoted 4 lines
mm/mlock.c +1 -1
-     if (!folio_test_large(folio))
+     if (likely(!folio_test_large(folio)))
              return 1;

How likely is this now? How likely will it be in two years time? Does this actually make any difference in either code generation or performance?

IMO, this hint could impact code generation and performance :) But it seems that 'likely' is not necessary here. I'll remove it.

Thanks again for your time!
Lance

Quoted 1 line
#3 lore raw
In reply to Lance Yang
Quoted 35 lines · Lance Yang

Let's make folio_mlock_step() simply a wrapper around folio_pte_batch(), which will greatly reduce the cost of ptep_get() when scanning a range of contptes.

---
 mm/mlock.c | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)
mm/mlock.c +1 -14
index 30b51cdea89d..1ae6232d38cf 100644
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
 static inline unsigned int folio_mlock_step(struct folio *folio,
                pte_t *pte, unsigned long addr, unsigned long end)
 {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
-       pte_t ptent = ptep_get(pte);
-
-       if (!folio_test_large(folio))
+       if (likely(!folio_test_large(folio)))
                return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)

mm/mlock.c void munlock_folio(struct folio *folio) · line 319

Quoted 10 lines
mm/mlock.c +5 -3
-                       break;
-       }
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       int max_nr = (end - addr) / PAGE_SIZE;
+       pte_t ptent = ptep_get(pte);
 
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, NULL,
+                              NULL, NULL);
 }

what about a minimum change as below?
index 30b51cdea89d..e8b98f84fbd2 100644

mm/mlock.c +4 -15
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
 static inline unsigned int folio_mlock_step(struct folio *folio,
                pte_t *pte, unsigned long addr, unsigned long end)
 {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
+       unsigned int count = (end - addr) >> PAGE_SHIFT;
        pte_t ptent = ptep_get(pte);
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
 
        if (!folio_test_large(folio))
                return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)
-                       break;
-       }
-
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, count, fpb_flags, NULL,
+                              NULL, NULL);
 }
 
 
Quoted 5 lines
mm/mlock.c +0 -1
 static inline bool allow_mlock_munlock(struct folio *folio,
--

2.33.1

author
#4 lore raw

Hi Barry,

Thanks for taking time to review!

Quoted 43 lines · Barry Song
Quoted 41 lines · Lance Yang

Let's make folio_mlock_step() simply a wrapper around folio_pte_batch(), which will greatly reduce the cost of ptep_get() when scanning a range of contptes.

---
 mm/mlock.c | 23 ++++++-----------------
 1 file changed, 6 insertions(+), 17 deletions(-)
mm/mlock.c +4 -16
index 30b51cdea89d..1ae6232d38cf 100644
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
 static inline unsigned int folio_mlock_step(struct folio *folio,
                pte_t *pte, unsigned long addr, unsigned long end)
 {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
-       pte_t ptent = ptep_get(pte);
-
-       if (!folio_test_large(folio))
+       if (likely(!folio_test_large(folio)))
                return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)
-                       break;
-       }
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       int max_nr = (end - addr) / PAGE_SIZE;
+       pte_t ptent = ptep_get(pte);
 
Quoted 6 lines

mm/mlock.c void munlock_folio(struct folio *folio) · line 319

Quoted 4 lines
mm/mlock.c +2 -1
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, NULL,
+                              NULL, NULL);
 }

what about a minimum change as below?

Nice, that makes sense to me ;)
I'll adjust as you suggested.

Thanks again for your time!
Lance

Quoted 40 lines

index 30b51cdea89d..e8b98f84fbd2 100644

mm/mlock.c +4 -15
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
 static inline unsigned int folio_mlock_step(struct folio *folio,
                pte_t *pte, unsigned long addr, unsigned long end)
 {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
+       unsigned int count = (end - addr) >> PAGE_SHIFT;
        pte_t ptent = ptep_get(pte);
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
 
        if (!folio_test_large(folio))
                return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)
-                       break;
-       }
-
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, count, fpb_flags, NULL,
+                              NULL, NULL);
 }
 
 
Quoted 6 lines
mm/mlock.c +0 -1
 static inline bool allow_mlock_munlock(struct folio *folio,
--

2.33.1

Reviewed-by
#5 lore raw
In reply to Barry Song
Quoted 70 lines · Barry Song
Quoted 35 lines · Lance Yang

Let's make folio_mlock_step() simply a wrapper around folio_pte_batch(), which will greatly reduce the cost of ptep_get() when scanning a range of contptes.

---
  mm/mlock.c | 23 ++++++-----------------
  1 file changed, 6 insertions(+), 17 deletions(-)
mm/mlock.c +1 -14
index 30b51cdea89d..1ae6232d38cf 100644
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
  static inline unsigned int folio_mlock_step(struct folio *folio,
                 pte_t *pte, unsigned long addr, unsigned long end)
  {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
-       pte_t ptent = ptep_get(pte);
-
-       if (!folio_test_large(folio))
+       if (likely(!folio_test_large(folio)))
                 return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)

mm/mlock.c void munlock_folio(struct folio *folio) · line 319

Quoted 10 lines
mm/mlock.c +5 -3
-                       break;
-       }
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       int max_nr = (end - addr) / PAGE_SIZE;
+       pte_t ptent = ptep_get(pte);
 
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, NULL,
+                              NULL, NULL);
  }

what about a minimum change as below?
index 30b51cdea89d..e8b98f84fbd2 100644

mm/mlock.c +2 -8
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
  static inline unsigned int folio_mlock_step(struct folio *folio,
                 pte_t *pte, unsigned long addr, unsigned long end)
  {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
+       unsigned int count = (end - addr) >> PAGE_SHIFT;
         pte_t ptent = ptep_get(pte);
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
 
         if (!folio_test_large(folio))
                 return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-

mm/mlock.c void munlock_folio(struct folio *folio) · line 319

Quoted 10 lines
mm/mlock.c +2 -7
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)
-                       break;
-       }
-
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, count, fpb_flags, NULL,
+                              NULL, NULL);
  }

LGTM.

#6 lore raw
Quoted 85 lines · Baolin Wang
Quoted 70 lines · Barry Song
Quoted 35 lines · Lance Yang

Let's make folio_mlock_step() simply a wrapper around folio_pte_batch(), which will greatly reduce the cost of ptep_get() when scanning a range of contptes.

---
   mm/mlock.c | 23 ++++++-----------------
   1 file changed, 6 insertions(+), 17 deletions(-)
mm/mlock.c +1 -14
index 30b51cdea89d..1ae6232d38cf 100644
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
   static inline unsigned int folio_mlock_step(struct folio *folio,
                  pte_t *pte, unsigned long addr, unsigned long end)
   {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
-       pte_t ptent = ptep_get(pte);
-
-       if (!folio_test_large(folio))
+       if (likely(!folio_test_large(folio)))
                  return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)

mm/mlock.c void munlock_folio(struct folio *folio) · line 319

Quoted 10 lines
mm/mlock.c +5 -3
-                       break;
-       }
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
+       int max_nr = (end - addr) / PAGE_SIZE;
+       pte_t ptent = ptep_get(pte);
 
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, max_nr, fpb_flags, NULL,
+                              NULL, NULL);
   }

what about a minimum change as below?
index 30b51cdea89d..e8b98f84fbd2 100644

mm/mlock.c +2 -8
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)
   static inline unsigned int folio_mlock_step(struct folio *folio,
                  pte_t *pte, unsigned long addr, unsigned long end)
   {
-       unsigned int count, i, nr = folio_nr_pages(folio);
-       unsigned long pfn = folio_pfn(folio);
+       unsigned int count = (end - addr) >> PAGE_SHIFT;
          pte_t ptent = ptep_get(pte);
+       const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY;
 
          if (!folio_test_large(folio))
                  return 1;
 
-       count = pfn + nr - pte_pfn(ptent);
-       count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT);
-
-       for (i = 0; i < count; i++, pte++) {
-               pte_t entry = ptep_get(pte);
-

mm/mlock.c void munlock_folio(struct folio *folio) · line 319

Quoted 10 lines
mm/mlock.c +2 -7
-               if (!pte_present(entry))
-                       break;
-               if (pte_pfn(entry) - pfn >= nr)
-                       break;
-       }
-
-       return i;
+       return folio_pte_batch(folio, addr, pte, ptent, count, fpb_flags, NULL,
+                              NULL, NULL);
   }

LGTM.

Quoted 2 lines
Signature · 3 lines
Cheers,

David / dhildenb