mm/mlock: implement folio_mlock_step() using folio_pte_batch()
Subsystems (2)
PATCH 1/1 mm/mlock: implement folio_mlock_step() using folio_pte_batch()
Let's make folio_mlock_step() simply a wrapper around folio_pte_batch(), which will greatly reduce the cost of ptep_get() when scanning a range of contptes.
--- mm/mlock.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)
mm/mlock.c +6 -17
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); -pte_t ptent =ptep_get (pte); - -if (!folio_test_large (folio)) +if (likely (!folio_test_large (folio)))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); - -if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr) -break ; - } +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ; +int max_nr = (end - addr) /PAGE_SIZE ; +pte_t ptent =ptep_get (pte); -return i; +return folio_pte_batch (folio, addr, pte, ptent, max_nr, fpb_flags,NULL , +NULL ,NULL ); } static inline bool allow_mlock_munlock(struct folio *folio,
Signature · 1 line
2.33.1
mm/mlock.c void munlock_folio(struct folio *folio) · line 312
Quoted 4 lines · Lance Yang
{
mm/mlock.c +0
-3-unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); -pte_t ptent =ptep_get (pte);
Please don't move type declarations later in the function. Just because you can doesn't mean you should.
mm/mlock.c void munlock_folio(struct folio *folio) · line 311
Quoted 3 lines
mm/mlock.c +1
-1-if (!folio_test_large (folio)) +if (likely (!folio_test_large (folio)))return 1 ;
How likely is this now? How likely will it be in two years time? Does this actually make any difference in either code generation or performance?
Hi Matthew,
Thanks for taking time to review!
Quoted 9 lines · Matthew Wilcox
mm/mlock.c void munlock_folio(struct folio *folio) · line 312
Quoted 4 lines · Lance Yang
{
mm/mlock.c +0
-3-unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); -pte_t ptent =ptep_get (pte);Please don't move type declarations later in the function. Just because you can doesn't mean you should.
Thanks for pointing this out, I'll adjust as you suggested.
Quoted 8 lines
mm/mlock.c void munlock_folio(struct folio *folio) · line 311
Quoted 4 lines
mm/mlock.c +1
-1-if (!folio_test_large (folio)) +if (likely (!folio_test_large (folio)))return 1 ;How likely is this now? How likely will it be in two years time? Does this actually make any difference in either code generation or performance?
IMO, this hint could impact code generation and performance :) But it seems that 'likely' is not necessary here. I'll remove it.
Thanks again for your time!
Lance
Quoted 1 line
Quoted 35 lines · Lance Yang
Let's make
folio_mlock_step()simply a wrapper aroundfolio_pte_batch(), which will greatly reduce the cost ofptep_get()when scanning a range of contptes.Signed-off-by Lance Yang <[email protected]>--- mm/mlock.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)mm/mlock.c +1
-14@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); -pte_t ptent =ptep_get (pte); - -if (!folio_test_large (folio)) +if (likely (!folio_test_large (folio)))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); - -if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr)
mm/mlock.c void munlock_folio(struct folio *folio) · line 319
Quoted 10 lines
mm/mlock.c +5
-3-break ; - } +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ; +int max_nr = (end - addr) /PAGE_SIZE ; +pte_t ptent =ptep_get (pte); -return i; +return folio_pte_batch (folio, addr, pte, ptent, max_nr, fpb_flags,NULL , +NULL ,NULL ); }
what about a minimum change as below?
index 30b51cdea89d..e8b98f84fbd2 100644
mm/mlock.c +4 -15
@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); +unsigned int count = (end - addr) >>PAGE_SHIFT ;pte_t ptent =ptep_get (pte); +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ;if (!folio_test_large (folio))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); - -if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr) -break ; - } - -return i; +return folio_pte_batch (folio, addr, pte, ptent, count, fpb_flags,NULL , +NULL ,NULL ); }
Quoted 5 lines
mm/mlock.c +0
-1static inline bool allow_mlock_munlock(struct folio *folio, --2.33.1
Hi Barry,
Thanks for taking time to review!
Quoted 43 lines · Barry Song
Quoted 41 lines · Lance Yang
Let's make
folio_mlock_step()simply a wrapper aroundfolio_pte_batch(), which will greatly reduce the cost ofptep_get()when scanning a range of contptes.Signed-off-by Lance Yang <[email protected]>--- mm/mlock.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)mm/mlock.c +4
-16@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); -pte_t ptent =ptep_get (pte); - -if (!folio_test_large (folio)) +if (likely (!folio_test_large (folio)))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); - -if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr) -break ; - } +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ; +int max_nr = (end - addr) /PAGE_SIZE ; +pte_t ptent =ptep_get (pte);
Quoted 6 lines
mm/mlock.c void munlock_folio(struct folio *folio) · line 319
Quoted 4 lines
mm/mlock.c +2
-1-return i; +return folio_pte_batch (folio, addr, pte, ptent, max_nr, fpb_flags,NULL , +NULL ,NULL ); }what about a minimum change as below?
Nice, that makes sense to me ;)
I'll adjust as you suggested.
Thanks again for your time!
Lance
Quoted 40 lines
index
30b51cdea89d..e8b98f84fbd2100644mm/mlock.c +4
-15@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); +unsigned int count = (end - addr) >>PAGE_SHIFT ;pte_t ptent =ptep_get (pte); +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ;if (!folio_test_large (folio))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); - -if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr) -break ; - } - -return i; +return folio_pte_batch (folio, addr, pte, ptent, count, fpb_flags,NULL , +NULL ,NULL ); }Quoted 6 lines
mm/mlock.c +0
-1static inline bool allow_mlock_munlock(struct folio *folio, --2.33.1
Baolin Wang
Reviewed-byQuoted 70 lines · Barry Song
Quoted 35 lines · Lance Yang
Let's make
folio_mlock_step()simply a wrapper aroundfolio_pte_batch(), which will greatly reduce the cost ofptep_get()when scanning a range of contptes.Signed-off-by Lance Yang <[email protected]>--- mm/mlock.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)mm/mlock.c +1
-14@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); -pte_t ptent =ptep_get (pte); - -if (!folio_test_large (folio)) +if (likely (!folio_test_large (folio)))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); - -if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr)mm/mlock.c void munlock_folio(struct folio *folio) · line 319
Quoted 10 lines
mm/mlock.c +5
-3-break ; - } +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ; +int max_nr = (end - addr) /PAGE_SIZE ; +pte_t ptent =ptep_get (pte); -return i; +return folio_pte_batch (folio, addr, pte, ptent, max_nr, fpb_flags,NULL , +NULL ,NULL ); }what about a minimum change as below?
index30b51cdea89d..e8b98f84fbd2100644mm/mlock.c +2
-8@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); +unsigned int count = (end - addr) >>PAGE_SHIFT ;pte_t ptent =ptep_get (pte); +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ;if (!folio_test_large (folio))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); -
mm/mlock.c void munlock_folio(struct folio *folio) · line 319
Quoted 10 lines
mm/mlock.c +2
-7-if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr) -break ; - } - -return i; +return folio_pte_batch (folio, addr, pte, ptent, count, fpb_flags,NULL , +NULL ,NULL ); }
LGTM.
David Hildenbrand
Acked-byQuoted 85 lines · Baolin Wang
Quoted 70 lines · Barry Song
Quoted 35 lines · Lance Yang
Let's make
folio_mlock_step()simply a wrapper aroundfolio_pte_batch(), which will greatly reduce the cost ofptep_get()when scanning a range of contptes.Signed-off-by Lance Yang <[email protected]>--- mm/mlock.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-)mm/mlock.c +1
-14@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); -pte_t ptent =ptep_get (pte); - -if (!folio_test_large (folio)) +if (likely (!folio_test_large (folio)))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); - -if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr)mm/mlock.c void munlock_folio(struct folio *folio) · line 319
Quoted 10 lines
mm/mlock.c +5
-3-break ; - } +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ; +int max_nr = (end - addr) /PAGE_SIZE ; +pte_t ptent =ptep_get (pte); -return i; +return folio_pte_batch (folio, addr, pte, ptent, max_nr, fpb_flags,NULL , +NULL ,NULL ); }what about a minimum change as below?
index30b51cdea89d..e8b98f84fbd2100644mm/mlock.c +2
-8@@ -307,26 +307,15 @@ void munlock_folio(struct folio *folio)static inline unsigned int folio_mlock_step (struct folio *folio,pte_t *pte,unsigned long addr,unsigned long end) { -unsigned int count, i, nr =folio_nr_pages (folio); -unsigned long pfn =folio_pfn (folio); +unsigned int count = (end - addr) >>PAGE_SHIFT ;pte_t ptent =ptep_get (pte); +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ;if (!folio_test_large (folio))return 1 ; - count = pfn + nr -pte_pfn (ptent); - count = min_t(unsigned int, count, (end - addr) >> PAGE_SHIFT); - -for (i =0 ; i < count; i++, pte++) { -pte_t entry =ptep_get (pte); -mm/mlock.c void munlock_folio(struct folio *folio) · line 319
Quoted 10 lines
mm/mlock.c +2
-7-if (!pte_present (entry)) -break ; -if (pte_pfn (entry) - pfn >= nr) -break ; - } - -return i; +return folio_pte_batch (folio, addr, pte, ptent, count, fpb_flags,NULL , +NULL ,NULL ); }LGTM.
Quoted 2 lines
Reviewed-by Baolin Wang <[email protected]>
Signature · 3 lines
Cheers, David / dhildenb