mempolicy: Optimize queue_folios_pte_range by PTE batching
Subsystems (2)
PATCH 1/1 mempolicy: Optimize queue_folios_pte_range by PTE batching
After the check for queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.
---
Unfortunately I have only build tested this since my test environment is broken.
mm/mempolicy.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)
mm/mempolicy.c +11 -1
@@ -573,6 +573,9 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,pte_t *pte, *mapped_pte;pte_t ptent;spinlock_t *ptl; +int max_nr; +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ; +int nr =1 ; ptl =pmd_trans_huge_lock (pmd, vma);if (ptl) {
@@ -586,7 +589,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, walk->action =ACTION_AGAIN ;return 0 ; } -for (; addr != end; pte++, addr +=PAGE_SIZE ) { +for (; addr != end; pte += nr, addr += nr *PAGE_SIZE ) { + nr =1 ; ptent =ptep_get (pte);if (pte_none (ptent))continue ;
@@ -607,6 +611,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,if (!queue_folio_required (folio, qp))continue ;if (folio_test_large (folio)) { + max_nr = (end - addr) >>PAGE_SHIFT ; +if (max_nr !=1 ) + nr =folio_pte_batch (folio, addr, pte, ptent, + max_nr, fpb_flags, +NULL ,NULL ,NULL );/* * A large folio can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may
@@ -633,6 +642,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, qp->nr_failed++;if (strictly_unmovable (flags))break ; + qp->nr_failed += nr -1 ; } }pte_unmap_unlock (mapped_pte, ptl);
Signature · 1 line
2.30.2
Quoted 17 lines · Dev Jain
After the check for
queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.Signed-off-by Dev Jain <[email protected]>---
Unfortunately I have only build tested this since my test environment is broken.mm/mempolicy.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)mm/mempolicy.c +0
-0
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, · line 578
Quoted 6 lines
mm/mempolicy.c +3
-0@@ -573,6 +573,9 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,pte_t *pte, *mapped_pte;pte_t ptent;spinlock_t *ptl; +int max_nr; +const fpb_t fpb_flags =FPB_IGNORE_DIRTY |FPB_IGNORE_SOFT_DIRTY ; +int nr =1 ;
Try sticking to reverse xmas tree, please. (not completely the case here, but fpb_flags can easily be moved all he way to the top)
Also, why are you initializing nr to 1 here if you reinitialize it below?
Quoted 12 lines
mm/mempolicy.c +1
-1ptl = pmd_trans_huge_lock(pmd, vma);> if (ptl) {@@ -586,7 +589,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, walk->action =ACTION_AGAIN ;return 0 ; } - for (; addr != end; pte++, addr += PAGE_SIZE) {> + for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) { + nr =1 ; ptent =ptep_get (pte);if (pte_none (ptent))continue ;
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, · line 618
Quoted 8 lines
mm/mempolicy.c +5
-0@@ -607,6 +611,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr,if (!queue_folio_required (folio, qp))continue ;if (folio_test_large (folio)) { + max_nr = (end - addr) >>PAGE_SHIFT ; +if (max_nr !=1 ) + nr =folio_pte_batch (folio, addr, pte, ptent, + max_nr, fpb_flags, +NULL ,NULL ,NULL );
We should probably do that immediately after we verified that vm_normal_folio() have us something reasonable.
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, · line 645
Quoted 8 lines
mm/mempolicy.c +1
-0/* * A large folio can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may @@ -633,6 +642,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, qp->nr_failed++;if (strictly_unmovable (flags))break ; + qp->nr_failed += nr -1 ;
Can't we do qp->nr_failed += nr; above?
Weird enough, queue_folios_pmd() also only does qp->nr_failed++, but queue_pages_range() documents it that way.
Quoted 3 lines
mm/mempolicy.c +0
-0} }pte_unmap_unlock (mapped_pte, ptl);
Signature · 3 lines
Cheers, David / dhildenb
Quoted 13 lines · David Hildenbrand
Quoted 12 lines · Dev Jain
After the check for
queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.Signed-off-by Dev Jain <[email protected]>---
Unfortunately I have only build tested this since my test environment is broken.
mm/mempolicy.c| 12 +++++++++++-1 file changed, 11 insertions(+), 1 deletion(-)
Quoted 14 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 579
Quoted 12 lines
mm/mempolicy.c +3
-0@@ -573,6 +573,9 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; + int max_nr; + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; + int nr = 1;Try sticking to reverse xmas tree, please. (not completely the case here, but
fpb_flagscan easily be moved all he way to the top)
I thought that the initializations were to be kept at the bottom.
Asking for future patches, should I put all declarations in reverse-xmas fashion (even those which I don't intend to touch w.r.t the patch logic), or do I do that for only my additions?
Quoted 2 lines
Also, why are you initializing nr to 1 here if you reinitialize it below?
Yup no need, I thought pte += nr will blow up due to nr not being initialized, but it won't because it gets executed just before the start of the second iteration.
Quoted 9 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 590
Quoted 8 lines
mm/mempolicy.c +0
-1ptl = pmd_trans_huge_lock(pmd, vma);> if (ptl) {@@ -586,7 +589,8 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) {> + for (;addr != end; pte += nr, addr += nr *
PAGE_SIZE) {
Quoted 16 lines
Quoted 5 lines
mm/mempolicy.c +1
-0+ nr = 1; ptent = ptep_get(pte); if (pte_none(ptent))continue;
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 619
Quoted 9 lines
mm/mempolicy.c +5
-0@@ -607,6 +611,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!queue_folio_required(folio, qp)) continue; if (folio_test_large(folio)) { + max_nr = (end - addr) >> PAGE_SHIFT; + if (max_nr != 1) + nr = folio_pte_batch(folio, addr, pte, ptent, + max_nr, fpb_flags, + NULL, NULL, NULL);We should probably do that immediately after we verified that
vm_normal_folio()have us something reasonable.
But shouldn't we keep the small folio case separate to avoid the overhead of folio_pte_batch()?
Quoted 12 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 646
Quoted 10 lines
mm/mempolicy.c +0
-0/** A large folio can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may
mm/mempolicy.c +1
-0@@ -633,6 +642,7 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, qp->nr_failed++; if (strictly_unmovable(flags)) break; + qp->nr_failed += nr - 1;Can't we do
qp->nr_failed+= nr; above?
I did not dive deep into the significance of nr_failed, but I did that to keep the code, before and after the change, equivalent:
Claim: if we reach qp->nr_failed++ for a single pte, we will reach here for all ptes belonging to the same batch.
Proof: We reach here => the if condition is true. Now, !(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) and !vma_migratable(vma) do not depend on the ptes. So the other case is that !migrate_folio_add() is true => !folio_isolate_lru() is true, which depends on the folio and not the PTEs; if isolation fails for one PTE, it will definitely fail for the PTE batch.
So, before the change, if we iterate on a pte mapping a large folio, and strictly_unmovable(flags) is true, then nr_failed += 1 only. If not, then nr_failed++ will happen nr times for sure (because of the claim) and we can safely do qp->nr_failed += nr - 1.
Quoted 8 lines
Weird enough,
queue_folios_pmd()also only does qp->nr_failed++, butqueue_pages_range()documents it that way.Quoted 3 lines
mm/mempolicy.c +0
-0} }
pte_unmap_unlock(mapped_pte, ptl);
Quoted 33 lines · Dev Jain
Quoted 14 lines · David Hildenbrand
Quoted 13 lines · Dev Jain
After the check for
queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this
loop by skipping over a PTE batch mapping the same folio.Signed-off-by Dev Jain <[email protected]>---
Unfortunately I have only build tested this since my test environment is broken.mm/mempolicy.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-)Quoted 15 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 579
Quoted 12 lines
mm/mempolicy.c +3
-0@@ -573,6 +573,9 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; + int max_nr; + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; + int nr = 1;Try sticking to reverse xmas tree, please. (not completely the case here, but
fpb_flagscan easily be moved all he way to the top)
Quoted 1 line
I thought that the initializations were to be kept at the bottom.
Not that I am aware of.
Quoted 3 lines
Asking for future patches, should I put all declarations in reverse-xmas fashion (even those which I don't intend to touch w.r.t the patch logic), or do I do that for only my additions?
We try to stay as close to reverse-xmas tree as possible. It's not always possible (e.g., dependent assignments), but fpb_flags in this case here can easily go all the way to the top.
...
Quoted 28 lines
Quoted 19 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 590
Quoted 9 lines
mm/mempolicy.c +0
-1ptl = pmd_trans_huge_lock(pmd, vma);> if (ptl) {@@ -586,7 +589,8 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) {> + for (;addr != end; pte += nr, addr += nr *
PAGE_SIZE) {Quoted 9 lines
mm/mempolicy.c +1
-0+ nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) continue;@@ -607,6 +611,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!queue_folio_required(folio, qp)) continue; if (folio_test_large(folio)) {Quoted 8 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, · line 618
Quoted 5 lines
mm/mempolicy.c +5
-0+ max_nr = (end - addr) >> PAGE_SHIFT; + if (max_nr != 1) + nr = folio_pte_batch(folio, addr, pte, ptent, + max_nr, fpb_flags, + NULL, NULL, NULL);We should probably do that immediately after we verified that
vm_normal_folio()have us something reasonable.
Quoted 2 lines
But shouldn't we keep the small folio case separate to avoid the overhead of
folio_pte_batch()?
Yes, just do something like
if (folio_test_large(folio) && end - addr > 1)
nr = folio_pte_batch(folio, addr, pte, ptent, end - addr, max_nr, fpb_flags, ...);
before the folio_test_reserved().
Then you'd also skip the all ptes if !queue_folio_required.
Quoted 14 lines
Quoted 13 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 646
Quoted 11 lines
mm/mempolicy.c +0
-0/** A large folio can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may
mm/mempolicy.c +1
-0@@ -633,6 +642,7 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, qp->nr_failed++; if (strictly_unmovable(flags)) break; + qp->nr_failed += nr - 1;Can't we do
qp->nr_failed+= nr; above?
Quoted 2 lines
I did not dive deep into the significance of
nr_failed, but I did that to keep the code, before and after the change, equivalent:
And I question exactly that.
If we hit strictly_unmovable(flags), we end up returning "-EIO" from queue_folios_pte_range().
And staring at queue_pages_range(), we ignore nr_failed if walk_page_range() returned an error.
So looks like we can just add everything in one shot, independent of strictly_unmovable()?
Signature · 3 lines
Cheers, David / dhildenb
Quoted 36 lines · David Hildenbrand
Quoted 33 lines · Dev Jain
Quoted 14 lines · David Hildenbrand
Quoted 13 lines · Dev Jain
After the check for
queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this
loop by skipping over a PTE batch mapping the same folio.Signed-off-by Dev Jain <[email protected]>---
Unfortunately I have only build tested this since my test environment is broken.
mm/mempolicy.c| 12 +++++++++++-1 file changed, 11 insertions(+), 1 deletion(-)Quoted 15 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 579
Quoted 12 lines
mm/mempolicy.c +3
-0@@ -573,6 +573,9 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; + int max_nr; + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; + int nr = 1;Try sticking to reverse xmas tree, please. (not completely the case here, but
fpb_flagscan easily be moved all he way to the top)Quoted 1 line
I thought that the initializations were to be kept at the bottom.
Quoted 7 lines
Not that I am aware of.
Quoted 3 lines
Asking for future patches, should I put all declarations in reverse-xmas fashion (even those which I don't intend to touch w.r.t the patch logic), or do I do that for only my additions?
We try to stay as close to reverse-xmas tree as possible. It's not always possible (e.g., dependent assignments), but
fpb_flagsin this case here can easily go all the way to the top.
Sure.
Quoted 36 lines
...
Quoted 28 lines
Quoted 19 lines
> ptl =
pmd_trans_huge_lock(pmd, vma);> if (ptl) {mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 592
Quoted 5 lines
mm/mempolicy.c +0
-0@@ -586,7 +589,8 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, walk->action = ACTION_AGAIN; return 0; }> - for (; addr != end; pte++, addr +=
PAGE_SIZE) {> + for (; addr != end; pte += nr, addr += nr *PAGE_SIZE) {Quoted 9 lines
mm/mempolicy.c +1
-0+ nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) continue;@@ -607,6 +611,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!queue_folio_required(folio, qp)) continue; if (folio_test_large(folio)) {Quoted 8 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, · line 618
Quoted 5 lines
mm/mempolicy.c +5
-0+ max_nr = (end - addr) >> PAGE_SHIFT; + if (max_nr != 1) + nr = folio_pte_batch(folio, addr, pte, ptent, + max_nr, fpb_flags, + NULL, NULL, NULL);We should probably do that immediately after we verified that
vm_normal_folio()have us something reasonable.Quoted 2 lines
But shouldn't we keep the small folio case separate to avoid the overhead of
folio_pte_batch()?Yes, just do something like
Quoted 7 lines
if (
folio_test_large(folio)&& end - addr > 1)
nr =folio_pte_batch(folio, addr, pte, ptent, end - addr, max_nr, fpb_flags, ...);before the
folio_test_reserved().Then you'd also skip the all ptes if !queue_folio_required.
Ah got you, thanks.
Quoted 18 lines
Quoted 15 lines
Quoted 14 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 646
Quoted 12 lines
mm/mempolicy.c +0
-0/** A large folio can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may
mm/mempolicy.c +1
-0@@ -633,6 +642,7 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, qp->nr_failed++; if (strictly_unmovable(flags)) break; + qp->nr_failed += nr - 1;Can't we do
qp->nr_failed+= nr; above?Quoted 2 lines
I did not dive deep into the significance of
nr_failed, but I did that to keep the code, before and after the change, equivalent:
Quoted 8 lines
And I question exactly that.
If we hit
strictly_unmovable(flags), we end up returning "-EIO" fromqueue_folios_pte_range().And staring at
queue_pages_range(), we ignorenr_failedifwalk_page_range()returned an error.So looks like we can just add everything in one shot, independent of
strictly_unmovable()?
Looks good to me this way. I'll change it, thanks.
Quoted 1 line
Quoted 79 lines · Dev Jain
Quoted 13 lines · David Hildenbrand
Quoted 12 lines · Dev Jain
After the check for
queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.Signed-off-by Dev Jain <[email protected]>---
Unfortunately I have only build tested this since my test environment is broken.
mm/mempolicy.c| 12 +++++++++++-1 file changed, 11 insertions(+), 1 deletion(-)Quoted 14 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 579
Quoted 12 lines
mm/mempolicy.c +3
-0@@ -573,6 +573,9 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; + int max_nr; + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; + int nr = 1;Try sticking to reverse xmas tree, please. (not completely the case here, but
fpb_flagscan easily be moved all he way to the top)I thought that the initializations were to be kept at the bottom.
Asking for future patches, should I put all declarations in reverse-xmas fashion (even those which I don't intend to touch w.r.t the patch logic), or do I do that for only my additions?Quoted 2 lines
Also, why are you initializing nr to 1 here if you reinitialize it below?
Yup no need, I thought pte += nr will blow up due to nr not being initialized, but it won't because it gets executed just before the start of the second iteration.
Quoted 8 lines
> ptl =
pmd_trans_huge_lock(pmd, vma);> if (ptl) {mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 592
Quoted 5 lines
mm/mempolicy.c +0
-0@@ -586,7 +589,8 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, walk->action = ACTION_AGAIN; return 0; }> - for (; addr != end; pte++, addr +=
PAGE_SIZE) {> + for (; addr != end; pte += nr, addr += nr *PAGE_SIZE) {Quoted 16 lines
Quoted 5 lines
mm/mempolicy.c +1
-0+ nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) continue;mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 619
Quoted 9 lines
mm/mempolicy.c +5
-0@@ -607,6 +611,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!queue_folio_required(folio, qp)) continue; if (folio_test_large(folio)) { + max_nr = (end - addr) >> PAGE_SHIFT; + if (max_nr != 1) + nr = folio_pte_batch(folio, addr, pte, ptent, + max_nr, fpb_flags, + NULL, NULL, NULL);We should probably do that immediately after we verified that
vm_normal_folio()have us something reasonable.But shouldn't we keep the small folio case separate to avoid the overhead of
folio_pte_batch()?Quoted 12 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 646
Quoted 10 lines
mm/mempolicy.c +0
-0/** A large folio can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may
mm/mempolicy.c +1
-0@@ -633,6 +642,7 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, qp->nr_failed++; if (strictly_unmovable(flags)) break; + qp->nr_failed += nr - 1;Can't we do
qp->nr_failed+= nr; above?
Quoted 4 lines
I did not dive deep into the significance of
nr_failed, but I did that to keep the code, before and after the change, equivalent:Claim: if we reach qp->nr_failed++ for a single pte, we will reach here for all ptes belonging to the same batch.
Sorry, I missed the previous discussion (I replied to your new version). I think this claim is incorrect, we will skip remaining ptes belonging to the same batch with checking 'qp->large'.
if (folio_test_large(folio)) {
if (folio == qp->large)
continue;
qp->large = folio;
}Quoted 87 lines · Baolin Wang
Quoted 79 lines · Dev Jain
Quoted 13 lines · David Hildenbrand
Quoted 12 lines · Dev Jain
After the check for
queue_folio_required(), the code only cares about the folio in the for loop, i.e the PTEs are redundant. Therefore, optimize this loop by skipping over a PTE batch mapping the same folio.Signed-off-by Dev Jain <[email protected]>---
Unfortunately I have only build tested this since my test environment is broken.
mm/mempolicy.c| 12 +++++++++++-1 file changed, 11 insertions(+), 1 deletion(-)Quoted 14 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 579
Quoted 12 lines
mm/mempolicy.c +3
-0@@ -573,6 +573,9 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; + int max_nr; + const fpb_t fpb_flags = FPB_IGNORE_DIRTY | FPB_IGNORE_SOFT_DIRTY; + int nr = 1;Try sticking to reverse xmas tree, please. (not completely the case here, but
fpb_flagscan easily be moved all he way to the top)I thought that the initializations were to be kept at the bottom.
Asking for future patches, should I put all declarations in reverse- xmas fashion (even those which I don't intend to touch w.r.t the patch logic), or do I do that for only my additions?Quoted 2 lines
Also, why are you initializing nr to 1 here if you reinitialize it below?
Yup no need, I thought pte += nr will blow up due to nr not being initialized, but it won't because it gets executed just before the start of the second iteration.
Quoted 8 lines
> ptl =
pmd_trans_huge_lock(pmd, vma);> if (ptl) {mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 592
Quoted 5 lines
mm/mempolicy.c +0
-0@@ -586,7 +589,8 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, walk->action = ACTION_AGAIN; return 0; }> - for (; addr != end; pte++, addr +=
PAGE_SIZE) {> + for (; addr != end; pte += nr, addr += nr *PAGE_SIZE) {Quoted 16 lines
Quoted 5 lines
mm/mempolicy.c +1
-0+ nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) continue;mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 619
Quoted 9 lines
mm/mempolicy.c +5
-0@@ -607,6 +611,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!queue_folio_required(folio, qp)) continue; if (folio_test_large(folio)) { + max_nr = (end - addr) >> PAGE_SHIFT; + if (max_nr != 1) + nr = folio_pte_batch(folio, addr, pte, ptent, + max_nr, fpb_flags, + NULL, NULL, NULL);We should probably do that immediately after we verified that
vm_normal_folio()have us something reasonable.But shouldn't we keep the small folio case separate to avoid the overhead of
folio_pte_batch()?Quoted 12 lines
mm/mempolicy.c static int queue_folios_pte_range(pmd_t *pmd, · line 646
Quoted 10 lines
mm/mempolicy.c +0
-0/** A large folio can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may
mm/mempolicy.c +1
-0@@ -633,6 +642,7 @@ static int queue_folios_pte_range(pmd_t *pmd,unsigned long addr, qp->nr_failed++; if (strictly_unmovable(flags)) break; + qp->nr_failed += nr - 1;Can't we do
qp->nr_failed+= nr; above?Quoted 4 lines
I did not dive deep into the significance of
nr_failed, but I did that to keep the code, before and after the change, equivalent:Claim: if we reach qp->nr_failed++ for a single pte, we will reach here for all ptes belonging to the same batch.
Quoted 7 lines
Sorry, I missed the previous discussion (I replied to your new version). I think this claim is incorrect, we will skip remaining ptes belonging to the same batch with checking '
qp->large'.if (
folio_test_large(folio)) {
if (folio ==qp->large)
continue;
qp->large= folio;
}
Oops you are right, I missed that.