mm/huge_memory: refactor __split_huge_pmd_locked()
Subsystems (2)
cover mm/huge_memory: refactor __split_huge_pmd_locked()
__split_huge_pmd_locked() currently decides from the VMA -- vma_is_anonymous(), then vma_is_special_huge() -- before inspecting the PMD entry, so split and zap can classify the same PMD differently. The entry itself should decide. The function also mixes the present and non-present cases, resulting in duplicated code.
This series uses the PMD entry and its folio to decide how to handle each PMD, as zap_huge_pmd() does. It separates the present and non-present paths and moves their common work into helper functions. After this change, __split_huge_pmd_locked() only selects the appropriate helper.
Yin Tirui (9):
mm/huge_memory: read the huge PMD entry once when splitting it mm/huge_memory: add and use huge_zero_pmd_can_split() mm/huge_memory: add and use unmap_huge_pmd_entry() mm/huge_memory: use normal_or_softleaf_folio_pmd() in the PMD split
path
mm/huge_memory: dispatch on the folio when splitting a huge PMD mm/huge_memory: add and use split_huge_pmd_anon_rmap() mm/huge_memory: add struct split_pmd_state mm/huge_memory: split present and non-present huge PMDs separately mm/huge_memory: unify the migration and device private PTE loops
mm/huge_memory.c | 525 +++++++++++++++++++++++++++-------------------- 1 file changed, 302 insertions(+), 223 deletions(-)
Signature · 1 line
2.34.1
RFC PATCH 1/9 mm/huge_memory: read the huge PMD entry once when splitting it
__split_huge_pmd_locked() re-reads *pmd several times to classify the same entry. The PMD page table lock is held throughout, so read it once into old_pmd and classify from that.
No functional change intended.
--- mm/huge_memory.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-)
mm/huge_memory.c +8 -8
@@ -3143,10 +3143,11 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, unsigned long haddr, bool freeze) {struct mm_struct *mm = vma->vm_mm; +pmd_t old_pmd = *pmd;struct folio *folio;struct page *page;pgtable_t pgtable; -pmd_t old_pmd, _pmd; +pmd_t _pmd;bool soft_dirty, uffd_wp = false, young = false, write = false;bool anon_exclusive = false, dirty = false;unsigned long addr;
@@ -3157,7 +3158,8 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,VM_BUG_ON_VMA (vma->vm_start > haddr, vma);VM_BUG_ON_VMA (vma->vm_end < haddr +HPAGE_PMD_SIZE , vma); -VM_WARN_ON_ONCE (!pmd_is_valid_softleaf (*pmd) && !pmd_trans_huge (*pmd)); +VM_WARN_ON_ONCE (!pmd_is_valid_softleaf (old_pmd) && + !pmd_trans_huge (old_pmd));count_vm_event (THP_SPLIT_PMD );
@@ -3193,7 +3195,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,return ; } -if (is_huge_zero_pmd (*pmd)) { +if (is_huge_zero_pmd (old_pmd)) {/* * FIXME: Do we want to invalidate secondary mmu by calling * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below
@@ -3206,10 +3208,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,return __split_huge_zero_page_pmd (vma, haddr, pmd); } -if (pmd_is_migration_entry (*pmd)) { +if (pmd_is_migration_entry (old_pmd)) {softleaf_t entry; - old_pmd = *pmd; entry =softleaf_from_pmd (old_pmd); page =softleaf_to_page (entry); folio =page_folio (page);
@@ -3222,10 +3223,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, anon_exclusive =softleaf_is_migration_read_exclusive (entry); young =softleaf_is_migration_young (entry); dirty =softleaf_is_migration_dirty (entry); - }else if (pmd_is_device_private_entry (*pmd)) { + }else if (pmd_is_device_private_entry (old_pmd)) {softleaf_t entry; - old_pmd = *pmd; entry =softleaf_from_pmd (old_pmd); page =softleaf_to_page (entry); folio =page_folio (page);
@@ -3417,7 +3417,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, }pte_unmap (pte); -if (!pmd_is_migration_entry (*pmd)) +if (!pmd_is_migration_entry (old_pmd))folio_remove_rmap_pmd (folio, page, vma);if (freeze)put_page (page);
Signature · 1 line
2.34.1
RFC PATCH 2/9 mm/huge_memory: add and use huge_zero_pmd_can_split()
In reply to Yin TiruiOnly a huge zero PMD in an anonymous VMA is split, into a page table of shared zero page mappings. Any other huge zero PMD is simply unmapped.
vm_normal_folio_pmd() returns NULL for a huge zero PMD, and the split path unmaps any entry which has no folio. So make the decision before the folio is looked up, in huge_zero_pmd_can_split().
No functional change intended.
--- mm/huge_memory.c | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-)
mm/huge_memory.c +23 -13
@@ -3139,6 +3139,15 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,pmd_populate (mm, pmd, pgtable); } +/* +* Only a huge zero PMD in an anonymous VMA is split, into a page table of +* shared zero page mappings. Any other huge zero PMD is simply unmapped. +*/ +static bool huge_zero_pmd_can_split (struct vm_area_struct *vma,pmd_t pmdval) +{ +return is_huge_zero_pmd (pmdval) &&vma_is_anonymous (vma); +} +static void __split_huge_pmd_locked (struct vm_area_struct *vma,pmd_t *pmd,unsigned long haddr,bool freeze) {
@@ -3163,6 +3172,20 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,count_vm_event (THP_SPLIT_PMD ); +/* +* FIXME: Do we want to invalidate secondary mmu by calling +* mmu_notifier_arch_invalidate_secondary_tlbs() see comments below +* inside __split_huge_pmd() ? +* +* We are going from a zero huge page write protected to zero small +* page also write protected so it does not seems useful to invalidate +* secondary mmu at this time. +*/ +if (huge_zero_pmd_can_split (vma, old_pmd)) { +__split_huge_zero_page_pmd (vma, haddr, pmd); +return ; + } +if (!vma_is_anonymous (vma)) { old_pmd =pmdp_huge_clear_flush (vma, haddr, pmd);/*
@@ -3195,19 +3218,6 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,return ; } -if (is_huge_zero_pmd (old_pmd)) { -/* -* FIXME: Do we want to invalidate secondary mmu by calling -* mmu_notifier_arch_invalidate_secondary_tlbs() see comments below -* inside __split_huge_pmd() ? -* -* We are going from a zero huge page write protected to zero -* small page also write protected so it does not seems useful -* to invalidate secondary mmu at this time. -*/ -return __split_huge_zero_page_pmd (vma, haddr, pmd); - } -if (pmd_is_migration_entry (old_pmd)) {softleaf_t entry;
Signature · 1 line
2.34.1
RFC PATCH 3/9 mm/huge_memory: add and use unmap_huge_pmd_entry()
In reply to Yin TiruiOnly a huge PMD entry mapping an anonymous folio is split. Everything else is unmapped and comes back on the next fault. Put that in unmap_huge_pmd_entry().
Old and new differ only for a device private entry in a non-anonymous VMA, which nothing in the tree can produce.
--- mm/huge_memory.c | 84 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 28 deletions(-)
mm/huge_memory.c +56 -28
@@ -3148,6 +3148,52 @@ static bool huge_zero_pmd_can_split(struct vm_area_struct *vma, pmd_t pmdval)return is_huge_zero_pmd (pmdval) &&vma_is_anonymous (vma); } +/** +* unmap_huge_pmd_entry() - Unmap a huge PMD entry rather than splitting it. +* @vma: The VMA @pmd belongs to. +* @haddr: The PMD-aligned address @pmd maps. +* @pmd: Pointer to the huge PMD entry. +* @folio: The folio @pmd describes, or NULL if it describes none. +* @is_present: Is @pmd a present entry rather than a softleaf entry? +* +* Only anonymous folios are rebuilt at PTE level when a huge PMD entry is +* split. Everything else is unmapped here and faulted back in on the next +* access. +*/ +static void unmap_huge_pmd_entry (struct vm_area_struct *vma, +unsigned long haddr,pmd_t *pmd,struct folio *folio, +bool is_present) +{ +struct mm_struct *mm = vma->vm_mm; +pmd_t old_pmd; + + old_pmd =pmdp_huge_clear_flush (vma, haddr, pmd); +/* +* We are going to unmap this huge page. So +* just go ahead and zap it +*/ +if (arch_needs_pgtable_deposit ()) +zap_deposited_table (mm, pmd); + +if (!folio) +return ; + +if (is_present) { +struct page *page =pmd_page (old_pmd); + +if (!folio_test_dirty (folio) &&pmd_dirty (old_pmd)) +folio_mark_dirty (folio); +if (!folio_test_referenced (folio) &&pmd_young (old_pmd)) +folio_set_referenced (folio); +folio_remove_rmap_pmd (folio, page, vma); + } + +add_mm_counter (mm,mm_counter_file (folio), -HPAGE_PMD_NR ); + +if (is_present) +folio_put (folio); +} +static void __split_huge_pmd_locked (struct vm_area_struct *vma,pmd_t *pmd,unsigned long haddr,bool freeze) {
@@ -3187,34 +3233,16 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, }if (!vma_is_anonymous (vma)) { - old_pmd =pmdp_huge_clear_flush (vma, haddr, pmd); -/* -* We are going to unmap this huge page. So -* just go ahead and zap it -*/ -if (arch_needs_pgtable_deposit ()) -zap_deposited_table (mm, pmd); -if (vma_is_special_huge (vma)) -return ; -if (unlikely (pmd_is_migration_entry (old_pmd))) { -const softleaf_t old_entry =softleaf_from_pmd (old_pmd); - - folio =softleaf_to_folio (old_entry); - }else if (is_huge_zero_pmd (old_pmd)) { -return ; - }else { - page =pmd_page (old_pmd); - folio =page_folio (page); -if (!folio_test_dirty (folio) &&pmd_dirty (old_pmd)) -folio_mark_dirty (folio); -if (!folio_test_referenced (folio) &&pmd_young (old_pmd)) -folio_set_referenced (folio); -folio_remove_rmap_pmd (folio, page, vma); -add_mm_counter (mm,mm_counter_file (folio), -HPAGE_PMD_NR ); -folio_put (folio); -return ; - } -add_mm_counter (mm,mm_counter_file (folio), -HPAGE_PMD_NR ); +const bool is_present =pmd_present (old_pmd); + +if (vma_is_special_huge (vma) ||is_huge_zero_pmd (old_pmd)) + folio =NULL ; +else if (is_present) + folio =page_folio (pmd_page (old_pmd)); +else + folio =softleaf_to_folio (softleaf_from_pmd (old_pmd)); + +unmap_huge_pmd_entry (vma, haddr, pmd, folio, is_present);return ; }
Signature · 1 line
2.34.1
RFC PATCH 4/9 mm/huge_memory: use normal_or_softleaf_folio_pmd() in the PMD split path
In reply to Yin TiruiGet the folio once with normal_or_softleaf_folio_pmd() and decide the deposit once with has_deposited_pgtable(), as zap_huge_pmd() does. That makes split and zap classify an entry the same way, and drops vma_is_special_huge() from this path.
Behaviour changes only where the entry and the VMA flags disagree, which no in-tree path produces.
--- mm/huge_memory.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-)
mm/huge_memory.c +12 -9
@@ -3172,7 +3172,7 @@ static void unmap_huge_pmd_entry(struct vm_area_struct *vma,* We are going to unmap this huge page. So * just go ahead and zap it */ -if (arch_needs_pgtable_deposit ()) +if (has_deposited_pgtable (vma, old_pmd, folio))zap_deposited_table (mm, pmd);if (!folio)
@@ -3199,6 +3199,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, {struct mm_struct *mm = vma->vm_mm;pmd_t old_pmd = *pmd; +const bool is_present =pmd_present (old_pmd);struct folio *folio;struct page *page;pgtable_t pgtable;
@@ -3232,16 +3233,18 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,return ; } -if (!vma_is_anonymous (vma)) { -const bool is_present =pmd_present (old_pmd); + folio =normal_or_softleaf_folio_pmd (vma, haddr, old_pmd, is_present); -if (vma_is_special_huge (vma) ||is_huge_zero_pmd (old_pmd)) - folio =NULL ; -else if (is_present) - folio =page_folio (pmd_page (old_pmd)); -else - folio =softleaf_to_folio (softleaf_from_pmd (old_pmd)); +/* +* A non-present entry which is neither a migration nor a device +* private entry is corrupt, and pmd_to_softleaf_folio() has already +* warned about it. Leave it alone rather than act on a PFN which +* means nothing. +*/ +if (unlikely (!is_present && !folio)) +return ; +if (!vma_is_anonymous (vma)) {unmap_huge_pmd_entry (vma, haddr, pmd, folio, is_present);return ; }
Signature · 1 line
2.34.1
Lance Yang
reviewerRe: [PATCH RFC 4/9] mm/huge_memory: use normal_or_softleaf_folio_pmd() in the PMD split path
Quoted 18 lines · Yin Tirui
Get the folio once with
normal_or_softleaf_folio_pmd()and decide the deposit once withhas_deposited_pgtable(), aszap_huge_pmd()does. That makes split and zap classify an entry the same way, and dropsvma_is_special_huge()from this path.Behaviour changes only where the entry and the VMA flags disagree, which no in-tree path produces.
Signed-off-by Yin Tirui <[email protected]>---
mm/huge_memory.c| 21 ++++++++++++---------1 file changed, 12 insertions(+), 9 deletions(-)mm/huge_memory.c +0
-0
mm/huge_memory.c static void unmap_huge_pmd_entry(struct vm_area_struct *vma, · line 3176
Quoted 6 lines
mm/huge_memory.c +1
-1@@ -3172,7 +3172,7 @@ static void unmap_huge_pmd_entry(struct vm_area_struct *vma,* We are going to unmap this huge page. So * just go ahead and zap it */ -if (arch_needs_pgtable_deposit ()) +if (has_deposited_pgtable (vma, old_pmd, folio))zap_deposited_table (mm, pmd);
Staring at this ... I think we're missing raw PFN mappings of the huge zero folio ...
static bool has_deposited_pgtable(struct vm_area_struct *vma, pmd_t pmdval,
struct folio *folio)
{
/* Some architectures require unconditional depositing. */
if (arch_needs_pgtable_deposit())
return true;...
if (is_huge_zero_pmd(pmdval)) return !vma_is_dax(vma);
...
return folio && folio_test_anon(folio); }
If a non-DAX raw PFN mapping on x86 (say, via mshv_vtl_low) matches huge_zero_pfn, vmf_insert_pfn_pmd() won't deposit a page table, but has_deposited_pgtable() will return true ...
Before this patch, arch_needs_pgtable_deposit() skips withdrawal when splitting this mapping on x86. The new check would instead reach pgtable_trans_huge_withdraw() and dereference NULL through pgtable->lru if pmd_huge_pte(mm, pmd) is NULL.
The helper already causes the same bug in zap_huge_pmd(), but extending it to split is a regression introduced by this patch. I'll send a separate fix, and we need it landed first :D
[...]
Cheers, Lance
Lance Yang
reviewerRe: [PATCH RFC 4/9] mm/huge_memory: use normal_or_softleaf_folio_pmd() in the PMD split path
Quoted 49 lines · Lance Yang
Quoted 18 lines · Yin Tirui
Get the folio once with
normal_or_softleaf_folio_pmd()and decide the deposit once withhas_deposited_pgtable(), aszap_huge_pmd()does. That makes split and zap classify an entry the same way, and dropsvma_is_special_huge()from this path.Behaviour changes only where the entry and the VMA flags disagree, which no in-tree path produces.
Signed-off-by Yin Tirui <[email protected]>---
mm/huge_memory.c| 21 ++++++++++++---------1 file changed, 12 insertions(+), 9 deletions(-)mm/huge_memory.c +0
-0mm/huge_memory.c static void unmap_huge_pmd_entry(struct vm_area_struct *vma, · line 3176
Quoted 6 lines
mm/huge_memory.c +1
-1@@ -3172,7 +3172,7 @@ static void unmap_huge_pmd_entry(struct vm_area_struct *vma,* We are going to unmap this huge page. So * just go ahead and zap it */ -if (arch_needs_pgtable_deposit ()) +if (has_deposited_pgtable (vma, old_pmd, folio))zap_deposited_table (mm, pmd);Staring at this ... I think we're missing raw PFN mappings of the huge zero folio ...
static bool has_deposited_pgtable(struct
vm_area_struct*vma,pmd_tpmdval,
struct folio *folio){ /* Some architectures require unconditional depositing. */ if (arch_needs_pgtable_deposit()) return true;...
if (is_huge_zero_pmd(pmdval)) return !vma_is_dax(vma);...
return folio && folio_test_anon(folio); }If a non-DAX raw PFN mapping on x86 (say, via
mshv_vtl_low) matcheshuge_zero_pfn,vmf_insert_pfn_pmd()won't deposit a page table, buthas_deposited_pgtable()will return true ...
Quoted 8 lines
Before this patch,
arch_needs_pgtable_deposit()skips withdrawal when splitting this mapping on x86. The new check would instead reachpgtable_trans_huge_withdraw()and dereference NULL throughpgtable->lruifpmd_huge_pte(mm, pmd)is NULL.The helper already causes the same bug in
zap_huge_pmd(), but extending it to split is a regression introduced by this patch. I'll send a separate fix, and we need it landed first :D
RFC PATCH 5/9 mm/huge_memory: dispatch on the folio when splitting a huge PMD
In reply to Yin TiruiOnly anonymous folios carry the anon rmap and PageAnonExclusive() state needed when rebuilding PTEs, so decide the split path from folio_test_anon() rather than vma_is_anonymous(). Treat a NULL folio as nothing to rebuild.
This only changes behavior for folio/VMA mismatches, which are not produced by any in-tree path.
--- mm/huge_memory.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-)
mm/huge_memory.c +2 -7
@@ -3244,7 +3244,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,if (unlikely (!is_present && !folio))return ; -if (!vma_is_anonymous (vma)) { +if (!folio || !folio_test_anon (folio)) {unmap_huge_pmd_entry (vma, haddr, pmd, folio, is_present);return ; }
@@ -3254,14 +3254,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, entry =softleaf_from_pmd (old_pmd); page =softleaf_to_page (entry); - folio =page_folio (page); soft_dirty =pmd_swp_soft_dirty (old_pmd); uffd_wp =pmd_swp_uffd (old_pmd); write =softleaf_is_migration_write (entry); -if (PageAnon (page)) - anon_exclusive =softleaf_is_migration_read_exclusive (entry); + anon_exclusive =softleaf_is_migration_read_exclusive (entry); young =softleaf_is_migration_young (entry); dirty =softleaf_is_migration_dirty (entry); }else if (pmd_is_device_private_entry (old_pmd)) {
@@ -3269,7 +3267,6 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, entry =softleaf_from_pmd (old_pmd); page =softleaf_to_page (entry); - folio =page_folio (page); soft_dirty =pmd_swp_soft_dirty (old_pmd); uffd_wp =pmd_swp_uffd (old_pmd);
@@ -3321,7 +3318,6 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,*/ old_pmd =pmdp_invalidate (vma, haddr, pmd); page =pmd_page (old_pmd); - folio =page_folio (page);if (pmd_dirty (old_pmd)) { dirty = true;folio_set_dirty (folio);
@@ -3332,7 +3328,6 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, uffd_wp =pmd_uffd (old_pmd);VM_WARN_ON_FOLIO (!folio_ref_count (folio), folio); -VM_WARN_ON_FOLIO (!folio_test_anon (folio), folio);/* * Without "freeze", we'll simply split the PMD, propagating the
Signature · 1 line
2.34.1
RFC PATCH 6/9 mm/huge_memory: add and use split_huge_pmd_anon_rmap()
In reply to Yin TiruiThe anon-exclusive handling and the PTE-level rmap conversion are written twice, once for present entries and once for device private ones. Factor them into one helper.
It returns whether the mapping may still be frozen instead of writing the caller's freeze back.
No functional change intended.
--- mm/huge_memory.c | 90 +++++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 43 deletions(-)
mm/huge_memory.c +47 -43
@@ -3194,6 +3194,43 @@ static void unmap_huge_pmd_entry(struct vm_area_struct *vma,folio_put (folio); } +/* +* Convert the folio's PMD-level anonymous rmap into PTE-level ones. +* +* Without "freeze", we'll simply split the PMD, propagating the +* PageAnonExclusive() flag for each PTE by setting it for +* each subpage -- no need to (temporarily) clear. +* +* With "freeze" we want to replace mapped pages by +* migration entries right away. This is only possible if we +* managed to clear PageAnonExclusive() -- see +* set_pmd_migration_entry(). +* +* In case we cannot clear PageAnonExclusive(), split the PMD +* only and let try_to_migrate_one() fail later. +* +* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. +* +* Returns: whether the mapping may still be frozen. +*/ +static bool split_huge_pmd_anon_rmap (struct folio *folio,struct page *page, +struct vm_area_struct *vma,unsigned long haddr,bool freeze, +bool anon_exclusive) +{ +rmap_t rmap_flags =RMAP_NONE ; + +if (freeze && + (!anon_exclusive || !folio_try_share_anon_rmap_pmd (folio, page))) +return true; + +folio_ref_add (folio,HPAGE_PMD_NR -1 ); +if (anon_exclusive) + rmap_flags |=RMAP_EXCLUSIVE ; +folio_add_anon_rmap_ptes (folio, page,HPAGE_PMD_NR , vma, haddr, + rmap_flags); +return false; +} +static void __split_huge_pmd_locked (struct vm_area_struct *vma,pmd_t *pmd,unsigned long haddr,bool freeze) {
@@ -3275,23 +3312,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, anon_exclusive =PageAnonExclusive (page);/* -* Device private THP should be treated the same as regular -* folios w.r.t anon exclusive handling. See the comments for -* folio handling and anon_exclusive below. +* Device private folios are treated the same as regular folios +* w.r.t. anon exclusive handling, see +* split_huge_pmd_anon_rmap(). */ -if (freeze && anon_exclusive && -folio_try_share_anon_rmap_pmd (folio, page)) - freeze = false; -if (!freeze) { -rmap_t rmap_flags =RMAP_NONE ; - -folio_ref_add (folio,HPAGE_PMD_NR -1 ); -if (anon_exclusive) - rmap_flags |=RMAP_EXCLUSIVE ; - -folio_add_anon_rmap_ptes (folio, page,HPAGE_PMD_NR , - vma, haddr, rmap_flags); - } + freeze =split_huge_pmd_anon_rmap (folio, page, vma, haddr, + freeze, anon_exclusive); }else {/* * Up to this point the pmd is present and huge and userland has
@@ -3315,6 +3341,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,* complete for this pmd), then we flush the SMP TLB and finally * we write the non-huge version of the pmd entry with * pmd_populate. +* +* This must also happen before PageAnonExclusive() is read +* below, see folio_try_share_anon_rmap_pmd(). */ old_pmd =pmdp_invalidate (vma, haddr, pmd); page =pmd_page (old_pmd);
@@ -3329,34 +3358,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,VM_WARN_ON_FOLIO (!folio_ref_count (folio), folio); -/* -* Without "freeze", we'll simply split the PMD, propagating the -* PageAnonExclusive() flag for each PTE by setting it for -* each subpage -- no need to (temporarily) clear. -* -* With "freeze" we want to replace mapped pages by -* migration entries right away. This is only possible if we -* managed to clear PageAnonExclusive() -- see -* set_pmd_migration_entry(). -* -* In case we cannot clear PageAnonExclusive(), split the PMD -* only and let try_to_migrate_one() fail later. -* -* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. -*/ anon_exclusive =PageAnonExclusive (page); -if (freeze && anon_exclusive && -folio_try_share_anon_rmap_pmd (folio, page)) - freeze = false; -if (!freeze) { -rmap_t rmap_flags =RMAP_NONE ; - -folio_ref_add (folio,HPAGE_PMD_NR -1 ); -if (anon_exclusive) - rmap_flags |=RMAP_EXCLUSIVE ; -folio_add_anon_rmap_ptes (folio, page,HPAGE_PMD_NR , - vma, haddr, rmap_flags); - } + freeze =split_huge_pmd_anon_rmap (folio, page, vma, haddr, + freeze, anon_exclusive); }/*
Signature · 1 line
2.34.1
RFC PATCH 7/9 mm/huge_memory: add struct split_pmd_state
In reply to Yin TiruiPut the state read out of the entry being split into one descriptor, so the read and write paths can be separated without passing a long argument list between them.
No functional change intended.
--- mm/huge_memory.c | 160 ++++++++++++++++++++++++++--------------------- 1 file changed, 88 insertions(+), 72 deletions(-)
mm/huge_memory.c +88 -72
@@ -3194,6 +3194,20 @@ static void unmap_huge_pmd_entry(struct vm_area_struct *vma,folio_put (folio); } +struct split_pmd_state { +struct folio *folio; +struct page *page; +bool is_present; +bool is_device_private; +bool freeze; +bool write; +bool young; +bool dirty; +bool soft_dirty; +bool uffd; +bool anon_exclusive; +}; +/* * Convert the folio's PMD-level anonymous rmap into PTE-level ones. *
@@ -3213,37 +3227,38 @@ static void unmap_huge_pmd_entry(struct vm_area_struct *vma,* * Returns: whether the mapping may still be frozen. */ -static bool split_huge_pmd_anon_rmap (struct folio *folio,struct page *page, -struct vm_area_struct *vma,unsigned long haddr,bool freeze, -bool anon_exclusive) +static bool split_huge_pmd_anon_rmap (const struct split_pmd_state *state, +struct vm_area_struct *vma,unsigned long haddr) {rmap_t rmap_flags =RMAP_NONE ; -if (freeze && - (!anon_exclusive || !folio_try_share_anon_rmap_pmd (folio, page))) +if (state->freeze && + (!state->anon_exclusive || + !folio_try_share_anon_rmap_pmd (state->folio, state->page)))return true; -folio_ref_add (folio,HPAGE_PMD_NR -1 ); -if (anon_exclusive) +folio_ref_add (state->folio,HPAGE_PMD_NR -1 ); +if (state->anon_exclusive) rmap_flags |=RMAP_EXCLUSIVE ; -folio_add_anon_rmap_ptes (folio, page,HPAGE_PMD_NR , vma, haddr, - rmap_flags); +folio_add_anon_rmap_ptes (state->folio, state->page,HPAGE_PMD_NR , vma, + haddr, rmap_flags);return false; }static void __split_huge_pmd_locked (struct vm_area_struct *vma,pmd_t *pmd,unsigned long haddr,bool freeze) { -struct mm_struct *mm = vma->vm_mm; -pmd_t old_pmd = *pmd; +const pmd_t old_pmd = *pmd;const bool is_present =pmd_present (old_pmd); +struct mm_struct *mm = vma->vm_mm; +struct split_pmd_state state = { + .is_present = is_present, + .freeze = freeze, + };struct folio *folio; -struct page *page; +unsigned long addr;pgtable_t pgtable;pmd_t _pmd; -bool soft_dirty, uffd_wp = false, young = false, write = false; -bool anon_exclusive = false, dirty = false; -unsigned long addr;pte_t *pte;int i;
@@ -3286,38 +3301,39 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,return ; } + state.folio = folio; +if (pmd_is_migration_entry (old_pmd)) { -softleaf_t entry; +const softleaf_t entry =softleaf_from_pmd (old_pmd); - entry =softleaf_from_pmd (old_pmd); - page =softleaf_to_page (entry); + state.page =softleaf_to_page (entry); - soft_dirty =pmd_swp_soft_dirty (old_pmd); - uffd_wp =pmd_swp_uffd (old_pmd); + state.soft_dirty =pmd_swp_soft_dirty (old_pmd); + state.uffd =pmd_swp_uffd (old_pmd); - write =softleaf_is_migration_write (entry); - anon_exclusive =softleaf_is_migration_read_exclusive (entry); - young =softleaf_is_migration_young (entry); - dirty =softleaf_is_migration_dirty (entry); + state.write =softleaf_is_migration_write (entry); + state.anon_exclusive = +softleaf_is_migration_read_exclusive (entry); + state.young =softleaf_is_migration_young (entry); + state.dirty =softleaf_is_migration_dirty (entry); }else if (pmd_is_device_private_entry (old_pmd)) { -softleaf_t entry; +const softleaf_t entry =softleaf_from_pmd (old_pmd); - entry =softleaf_from_pmd (old_pmd); - page =softleaf_to_page (entry); + state.is_device_private = true; + state.page =softleaf_to_page (entry); - soft_dirty =pmd_swp_soft_dirty (old_pmd); - uffd_wp =pmd_swp_uffd (old_pmd); + state.soft_dirty =pmd_swp_soft_dirty (old_pmd); + state.uffd =pmd_swp_uffd (old_pmd); - write =softleaf_is_device_private_write (entry); - anon_exclusive =PageAnonExclusive (page); + state.write =softleaf_is_device_private_write (entry); + state.anon_exclusive =PageAnonExclusive (state.page);/* * Device private folios are treated the same as regular folios * w.r.t. anon exclusive handling, see * split_huge_pmd_anon_rmap(). */ - freeze =split_huge_pmd_anon_rmap (folio, page, vma, haddr, - freeze, anon_exclusive); + state.freeze =split_huge_pmd_anon_rmap (&state, vma, haddr); }else {/* * Up to this point the pmd is present and huge and userland has
@@ -3345,22 +3361,22 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,* This must also happen before PageAnonExclusive() is read * below, see folio_try_share_anon_rmap_pmd(). */ - old_pmd =pmdp_invalidate (vma, haddr, pmd); - page =pmd_page (old_pmd); -if (pmd_dirty (old_pmd)) { - dirty = true; +const pmd_t pmdval =pmdp_invalidate (vma, haddr, pmd); + + state.page =pmd_page (pmdval); + state.write =pmd_write (pmdval); + state.young =pmd_young (pmdval); + state.dirty =pmd_dirty (pmdval); + state.soft_dirty =pmd_soft_dirty (pmdval); + state.uffd =pmd_uffd (pmdval); + state.anon_exclusive =PageAnonExclusive (state.page); + +if (state.dirty)folio_set_dirty (folio); - } - write =pmd_write (old_pmd); - young =pmd_young (old_pmd); - soft_dirty =pmd_soft_dirty (old_pmd); - uffd_wp =pmd_uffd (old_pmd);VM_WARN_ON_FOLIO (!folio_ref_count (folio), folio); - anon_exclusive =PageAnonExclusive (page); - freeze =split_huge_pmd_anon_rmap (folio, page, vma, haddr, - freeze, anon_exclusive); + state.freeze =split_huge_pmd_anon_rmap (&state, vma, haddr); }/*
@@ -3377,33 +3393,33 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,* Note that NUMA hinting access restrictions are not transferred to * avoid any possibility of altering permissions across VMAs. */ -if (freeze ||pmd_is_migration_entry (old_pmd)) { +if (state.freeze || (!state.is_present && !state.is_device_private)) {pte_t entry;swp_entry_t swp_entry;for (i =0 , addr = haddr; i <HPAGE_PMD_NR ; i++, addr +=PAGE_SIZE ) { -if (write) +if (state.write) swp_entry =make_writable_migration_entry ( -page_to_pfn (page + i)); -else if (anon_exclusive) +page_to_pfn (state.page + i)); +else if (state.anon_exclusive) swp_entry =make_readable_exclusive_migration_entry ( -page_to_pfn (page + i)); +page_to_pfn (state.page + i));else swp_entry =make_readable_migration_entry ( -page_to_pfn (page + i)); -if (young) +page_to_pfn (state.page + i)); +if (state.young) swp_entry =make_migration_entry_young (swp_entry); -if (dirty) +if (state.dirty) swp_entry =make_migration_entry_dirty (swp_entry); entry =swp_entry_to_pte (swp_entry); -if (soft_dirty) +if (state.soft_dirty) entry =pte_swp_mksoft_dirty (entry); -if (uffd_wp) +if (state.uffd) entry =pte_swp_mkuffd (entry);VM_WARN_ON (!pte_none (ptep_get (pte + i)));set_pte_at (mm, addr, pte + i, entry); } - }else if (pmd_is_device_private_entry (old_pmd)) { + }else if (state.is_device_private) {pte_t entry;swp_entry_t swp_entry;
@@ -3413,19 +3429,19 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,* pages corresponding to the pte entries when freeze * is false. */ -if (write) +if (state.write) swp_entry =make_writable_device_private_entry ( -page_to_pfn (page + i)); +page_to_pfn (state.page + i));else swp_entry =make_readable_device_private_entry ( -page_to_pfn (page + i)); +page_to_pfn (state.page + i));/* * Young and dirty bits are not progated via swp_entry */ entry =swp_entry_to_pte (swp_entry); -if (soft_dirty) +if (state.soft_dirty) entry =pte_swp_mksoft_dirty (entry); -if (uffd_wp) +if (state.uffd) entry =pte_swp_mkuffd (entry);VM_WARN_ON (!pte_none (ptep_get (pte + i)));set_pte_at (mm, addr, pte + i, entry);
@@ -3433,21 +3449,21 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, }else {pte_t entry; - entry =mk_pte (page,READ_ONCE (vma->vm_page_prot)); -if (write) + entry =mk_pte (state.page,READ_ONCE (vma->vm_page_prot)); +if (state.write) entry =pte_mkwrite (entry, vma); -if (!young) +if (!state.young) entry =pte_mkold (entry);/* NOTE: this may set soft-dirty too on some archs */ -if (dirty) +if (state.dirty) entry =pte_mkdirty (entry); -if (soft_dirty) +if (state.soft_dirty) entry =pte_mksoft_dirty (entry); -if (uffd_wp) +if (state.uffd) entry =pte_mkuffd (entry);/* Restore PAGE_NONE so an RWP marker keeps trapping */ -if (userfaultfd_rwp (vma) && uffd_wp) +if (userfaultfd_rwp (vma) && state.uffd) entry =pte_modify (entry,PAGE_NONE );for (i =0 ; i <HPAGE_PMD_NR ; i++)
@@ -3457,10 +3473,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, }pte_unmap (pte); -if (!pmd_is_migration_entry (old_pmd)) -folio_remove_rmap_pmd (folio, page, vma); -if (freeze) -put_page (page); +if (state.is_present || state.is_device_private) +folio_remove_rmap_pmd (state.folio, state.page, vma); +if (state.freeze) +put_page (state.page);smp_wmb ();/* make pte visible before pmd */ pmd_populate (mm, pmd, pgtable);
Signature · 1 line
2.34.1
RFC PATCH 8/9 mm/huge_memory: split present and non-present huge PMDs separately
In reply to Yin TiruiMove the shared PTE rebuild into split_huge_pmd_to_ptes(), then add split_present_huge_pmd() and split_non_present_huge_pmd() on top of it to separate the present and non-present cases in __split_huge_pmd_locked().
No functional change intended.
--- mm/huge_memory.c | 334 +++++++++++++++++++++++++---------------------- 1 file changed, 180 insertions(+), 154 deletions(-)
mm/huge_memory.c +180 -154
@@ -3245,143 +3245,31 @@ static bool split_huge_pmd_anon_rmap(const struct split_pmd_state *state,return false; } -static void __split_huge_pmd_locked (struct vm_area_struct *vma,pmd_t *pmd, -unsigned long haddr,bool freeze) +/* +* Replace an anonymous huge PMD entry with a page table mapping the same +* folio at PTE granularity. +*/ +static void split_huge_pmd_to_ptes (struct vm_area_struct *vma, +unsigned long haddr,pmd_t *pmd,struct split_pmd_state *state) { -const pmd_t old_pmd = *pmd; -const bool is_present =pmd_present (old_pmd); +/* Present mappings and device private entries hold a PMD-level rmap. */ +const bool rmapped = state->is_present || state->is_device_private;struct mm_struct *mm = vma->vm_mm; -struct split_pmd_state state = { - .is_present = is_present, - .freeze = freeze, - }; -struct folio *folio; +struct page *page = state->page;unsigned long addr;pgtable_t pgtable;pmd_t _pmd;pte_t *pte;int i; -VM_BUG_ON (haddr & ~HPAGE_PMD_MASK ); -VM_BUG_ON_VMA (vma->vm_start > haddr, vma); -VM_BUG_ON_VMA (vma->vm_end < haddr +HPAGE_PMD_SIZE , vma); - -VM_WARN_ON_ONCE (!pmd_is_valid_softleaf (old_pmd) && - !pmd_trans_huge (old_pmd)); - -count_vm_event (THP_SPLIT_PMD ); +if (rmapped) + state->freeze =split_huge_pmd_anon_rmap (state, vma, haddr);/* -* FIXME: Do we want to invalidate secondary mmu by calling -* mmu_notifier_arch_invalidate_secondary_tlbs() see comments below -* inside __split_huge_pmd() ? -* -* We are going from a zero huge page write protected to zero small -* page also write protected so it does not seems useful to invalidate -* secondary mmu at this time. -*/ -if (huge_zero_pmd_can_split (vma, old_pmd)) { -__split_huge_zero_page_pmd (vma, haddr, pmd); -return ; - } - - folio =normal_or_softleaf_folio_pmd (vma, haddr, old_pmd, is_present); - -/* -* A non-present entry which is neither a migration nor a device -* private entry is corrupt, and pmd_to_softleaf_folio() has already -* warned about it. Leave it alone rather than act on a PFN which -* means nothing. -*/ -if (unlikely (!is_present && !folio)) -return ; - -if (!folio || !folio_test_anon (folio)) { -unmap_huge_pmd_entry (vma, haddr, pmd, folio, is_present); -return ; - } - - state.folio = folio; - -if (pmd_is_migration_entry (old_pmd)) { -const softleaf_t entry =softleaf_from_pmd (old_pmd); - - state.page =softleaf_to_page (entry); - - state.soft_dirty =pmd_swp_soft_dirty (old_pmd); - state.uffd =pmd_swp_uffd (old_pmd); - - state.write =softleaf_is_migration_write (entry); - state.anon_exclusive = -softleaf_is_migration_read_exclusive (entry); - state.young =softleaf_is_migration_young (entry); - state.dirty =softleaf_is_migration_dirty (entry); - }else if (pmd_is_device_private_entry (old_pmd)) { -const softleaf_t entry =softleaf_from_pmd (old_pmd); - - state.is_device_private = true; - state.page =softleaf_to_page (entry); - - state.soft_dirty =pmd_swp_soft_dirty (old_pmd); - state.uffd =pmd_swp_uffd (old_pmd); - - state.write =softleaf_is_device_private_write (entry); - state.anon_exclusive =PageAnonExclusive (state.page); - -/* -* Device private folios are treated the same as regular folios -* w.r.t. anon exclusive handling, see -* split_huge_pmd_anon_rmap(). -*/ - state.freeze =split_huge_pmd_anon_rmap (&state, vma, haddr); - }else { -/* -* Up to this point the pmd is present and huge and userland has -* the whole access to the hugepage during the split (which -* happens in place). If we overwrite the pmd with the not-huge -* version pointing to the pte here (which of course we could if -* all CPUs were bug free), userland could trigger a small page -* size TLB miss on the small sized TLB while the hugepage TLB -* entry is still established in the huge TLB. Some CPU doesn't -* like that. See -* http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum -* 383 on page 105. Intel should be safe but is also warns that -* it's only safe if the permission and cache attributes of the -* two entries loaded in the two TLB is identical (which should -* be the case here). But it is generally safer to never allow -* small and huge TLB entries for the same virtual address to be -* loaded simultaneously. So instead of doing "pmd_populate(); -* flush_pmd_tlb_range();" we first mark the current pmd -* notpresent (atomically because here the pmd_trans_huge must -* remain set at all times on the pmd until the split is -* complete for this pmd), then we flush the SMP TLB and finally -* we write the non-huge version of the pmd entry with -* pmd_populate. -* -* This must also happen before PageAnonExclusive() is read -* below, see folio_try_share_anon_rmap_pmd(). -*/ -const pmd_t pmdval =pmdp_invalidate (vma, haddr, pmd); - - state.page =pmd_page (pmdval); - state.write =pmd_write (pmdval); - state.young =pmd_young (pmdval); - state.dirty =pmd_dirty (pmdval); - state.soft_dirty =pmd_soft_dirty (pmdval); - state.uffd =pmd_uffd (pmdval); - state.anon_exclusive =PageAnonExclusive (state.page); - -if (state.dirty) -folio_set_dirty (folio); - -VM_WARN_ON_FOLIO (!folio_ref_count (folio), folio); - - state.freeze =split_huge_pmd_anon_rmap (&state, vma, haddr); - } - -/* -* Withdraw the table only after we mark the pmd entry invalid. -* This's critical for some architectures (Power). +* The caller has already invalidated a present entry, and a softleaf +* entry is not present to begin with. Either way the entry is out of +* service before we withdraw the deposited page table, which is +* critical for some architectures (Power). */ pgtable =pgtable_trans_huge_withdraw (mm, pmd);pmd_populate (mm, &_pmd, pgtable);
@@ -3393,33 +3281,34 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,* Note that NUMA hinting access restrictions are not transferred to * avoid any possibility of altering permissions across VMAs. */ -if (state.freeze || (!state.is_present && !state.is_device_private)) { +if (state->freeze || + (!state->is_present && !state->is_device_private)) {pte_t entry;swp_entry_t swp_entry;for (i =0 , addr = haddr; i <HPAGE_PMD_NR ; i++, addr +=PAGE_SIZE ) { -if (state.write) +if (state->write) swp_entry =make_writable_migration_entry ( -page_to_pfn (state.page + i)); -else if (state.anon_exclusive) +page_to_pfn (page + i)); +else if (state->anon_exclusive) swp_entry =make_readable_exclusive_migration_entry ( -page_to_pfn (state.page + i)); +page_to_pfn (page + i));else swp_entry =make_readable_migration_entry ( -page_to_pfn (state.page + i)); -if (state.young) +page_to_pfn (page + i)); +if (state->young) swp_entry =make_migration_entry_young (swp_entry); -if (state.dirty) +if (state->dirty) swp_entry =make_migration_entry_dirty (swp_entry); entry =swp_entry_to_pte (swp_entry); -if (state.soft_dirty) +if (state->soft_dirty) entry =pte_swp_mksoft_dirty (entry); -if (state.uffd) +if (state->uffd) entry =pte_swp_mkuffd (entry);VM_WARN_ON (!pte_none (ptep_get (pte + i)));set_pte_at (mm, addr, pte + i, entry); } - }else if (state.is_device_private) { + }else if (state->is_device_private) {pte_t entry;swp_entry_t swp_entry;
@@ -3429,19 +3318,19 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,* pages corresponding to the pte entries when freeze * is false. */ -if (state.write) +if (state->write) swp_entry =make_writable_device_private_entry ( -page_to_pfn (state.page + i)); +page_to_pfn (page + i));else swp_entry =make_readable_device_private_entry ( -page_to_pfn (state.page + i)); +page_to_pfn (page + i));/* * Young and dirty bits are not progated via swp_entry */ entry =swp_entry_to_pte (swp_entry); -if (state.soft_dirty) +if (state->soft_dirty) entry =pte_swp_mksoft_dirty (entry); -if (state.uffd) +if (state->uffd) entry =pte_swp_mkuffd (entry);VM_WARN_ON (!pte_none (ptep_get (pte + i)));set_pte_at (mm, addr, pte + i, entry);
@@ -3449,21 +3338,21 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, }else {pte_t entry; - entry =mk_pte (state.page,READ_ONCE (vma->vm_page_prot)); -if (state.write) + entry =mk_pte (page,READ_ONCE (vma->vm_page_prot)); +if (state->write) entry =pte_mkwrite (entry, vma); -if (!state.young) +if (!state->young) entry =pte_mkold (entry);/* NOTE: this may set soft-dirty too on some archs */ -if (state.dirty) +if (state->dirty) entry =pte_mkdirty (entry); -if (state.soft_dirty) +if (state->soft_dirty) entry =pte_mksoft_dirty (entry); -if (state.uffd) +if (state->uffd) entry =pte_mkuffd (entry);/* Restore PAGE_NONE so an RWP marker keeps trapping */ -if (userfaultfd_rwp (vma) && state.uffd) +if (userfaultfd_rwp (vma) && state->uffd) entry =pte_modify (entry,PAGE_NONE );for (i =0 ; i <HPAGE_PMD_NR ; i++)
@@ -3473,15 +3362,152 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, }pte_unmap (pte); -if (state.is_present || state.is_device_private) -folio_remove_rmap_pmd (state.folio, state.page, vma); -if (state.freeze) -put_page (state.page); +if (rmapped) +folio_remove_rmap_pmd (state->folio, page, vma); +if (state->freeze) +put_page (page);smp_wmb ();/* make pte visible before pmd */ pmd_populate (mm, pmd, pgtable); } +static void split_present_huge_pmd (struct vm_area_struct *vma, +unsigned long haddr,pmd_t *pmd,struct folio *folio, +bool freeze) +{ +struct split_pmd_state state = { + .folio = folio, + .is_present = true, + .freeze = freeze, + }; + +/* +* Up to this point the pmd is present and huge and userland has the +* whole access to the hugepage during the split (which happens in +* place). If we overwrite the pmd with the not-huge version pointing +* to the pte here (which of course we could if all CPUs were bug +* free), userland could trigger a small page size TLB miss on the +* small sized TLB while the hugepage TLB entry is still established in +* the huge TLB. Some CPU doesn't like that. See +* http://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf, Erratum 383 on +* page 105. Intel should be safe but is also warns that it's only safe +* if the permission and cache attributes of the two entries loaded in +* the two TLB is identical (which should be the case here). But it is +* generally safer to never allow small and huge TLB entries for the +* same virtual address to be loaded simultaneously. So instead of +* doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the +* current pmd notpresent (atomically because here the pmd_trans_huge +* must remain set at all times on the pmd until the split is complete +* for this pmd), then we flush the SMP TLB and finally we write the +* non-huge version of the pmd entry with pmd_populate. +* +* This must also happen before PageAnonExclusive() is read below, see +* folio_try_share_anon_rmap_pmd(). +*/ +const pmd_t pmdval =pmdp_invalidate (vma, haddr, pmd); + + state.page =pmd_page (pmdval); + state.write =pmd_write (pmdval); + state.young =pmd_young (pmdval); + state.dirty =pmd_dirty (pmdval); + state.soft_dirty =pmd_soft_dirty (pmdval); + state.uffd =pmd_uffd (pmdval); + state.anon_exclusive =PageAnonExclusive (state.page); + +if (state.dirty) +folio_set_dirty (folio); + +VM_WARN_ON_FOLIO (!folio_ref_count (folio), folio); + +split_huge_pmd_to_ptes (vma, haddr, pmd, &state); +} + +static void split_non_present_huge_pmd (struct vm_area_struct *vma, +unsigned long haddr,pmd_t *pmd,pmd_t old_pmd, +struct folio *folio,bool freeze) +{ +const softleaf_t entry =softleaf_from_pmd (old_pmd); +struct split_pmd_state state = { + .folio = folio, + .page =softleaf_to_page (entry), + .is_device_private =softleaf_is_device_private (entry), + .freeze = freeze, + .soft_dirty =pmd_swp_soft_dirty (old_pmd), + .uffd =pmd_swp_uffd (old_pmd), + }; + +if (state.is_device_private) { +/* +* Device private folios are treated the same as regular folios +* w.r.t. anon exclusive handling, see +* split_huge_pmd_anon_rmap(). +*/ + state.write =softleaf_is_device_private_write (entry); + state.anon_exclusive =PageAnonExclusive (state.page); + }else { + state.write =softleaf_is_migration_write (entry); + state.young =softleaf_is_migration_young (entry); + state.dirty =softleaf_is_migration_dirty (entry); + state.anon_exclusive = +softleaf_is_migration_read_exclusive (entry); + } + +split_huge_pmd_to_ptes (vma, haddr, pmd, &state); +} + +static void __split_huge_pmd_locked (struct vm_area_struct *vma,pmd_t *pmd, +unsigned long haddr,bool freeze) +{ +const pmd_t old_pmd = *pmd; +const bool is_present =pmd_present (old_pmd); +struct folio *folio; + +VM_BUG_ON (haddr & ~HPAGE_PMD_MASK ); +VM_BUG_ON_VMA (vma->vm_start > haddr, vma); +VM_BUG_ON_VMA (vma->vm_end < haddr +HPAGE_PMD_SIZE , vma); + +VM_WARN_ON_ONCE (!pmd_is_valid_softleaf (old_pmd) && + !pmd_trans_huge (old_pmd)); + +count_vm_event (THP_SPLIT_PMD ); + +/* +* FIXME: Do we want to invalidate secondary mmu by calling +* mmu_notifier_arch_invalidate_secondary_tlbs() see comments below +* inside __split_huge_pmd() ? +* +* We are going from a zero huge page write protected to zero small +* page also write protected so it does not seems useful to invalidate +* secondary mmu at this time. +*/ +if (huge_zero_pmd_can_split (vma, old_pmd)) { +__split_huge_zero_page_pmd (vma, haddr, pmd); +return ; + } + + folio =normal_or_softleaf_folio_pmd (vma, haddr, old_pmd, is_present); + +/* +* A non-present entry which is neither a migration nor a device +* private entry is corrupt, and pmd_to_softleaf_folio() has already +* warned about it. Leave it alone rather than act on a PFN which +* means nothing. +*/ +if (unlikely (!is_present && !folio)) +return ; + +if (!folio || !folio_test_anon (folio)) { +unmap_huge_pmd_entry (vma, haddr, pmd, folio, is_present); +return ; + } + +if (is_present) +split_present_huge_pmd (vma, haddr, pmd, folio, freeze); +else +split_non_present_huge_pmd (vma, haddr, pmd, old_pmd, folio, + freeze); +} +void split_huge_pmd_locked (struct vm_area_struct *vma,unsigned long address,pmd_t *pmd,bool freeze) {
Signature · 1 line
2.34.1
RFC PATCH 9/9 mm/huge_memory: unify the migration and device private PTE loops
In reply to Yin TiruiWhen splitting a huge PMD into non-present PTEs, one loop installs migration entries while the other installs device private entries. They differ only in the leaf entry they build. Build the entry in split_pmd_make_softleaf() and use one loop.
Freezing still installs migration entries even for a device private mapping, as before.
No functional change intended.
--- mm/huge_memory.c | 89 +++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 46 deletions(-)
mm/huge_memory.c +43 -46
@@ -3245,6 +3245,42 @@ static bool split_huge_pmd_anon_rmap(const struct split_pmd_state *state,return false; } +/* +* Build the leaf entry for the PTE entry describing @pfn, for a huge PMD entry +* which is not restored as a present mapping. +*/ +static softleaf_t split_pmd_make_softleaf (const struct split_pmd_state *state, +unsigned long pfn) +{ +softleaf_t entry; + +if (state->is_device_private && !state->freeze) { +/* +* anon_exclusive was already propagated to the pages backing +* the PTE entries by split_huge_pmd_anon_rmap(), and accessed +* and dirty bits are not propagated via device private +* entries. +*/ +if (state->write) +return make_writable_device_private_entry (pfn); +return make_readable_device_private_entry (pfn); + } + +if (state->write) + entry =make_writable_migration_entry (pfn); +else if (state->anon_exclusive) + entry =make_readable_exclusive_migration_entry (pfn); +else + entry =make_readable_migration_entry (pfn); + +if (state->young) + entry =make_migration_entry_young (entry); +if (state->dirty) + entry =make_migration_entry_dirty (entry); + +return entry; +} +/* * Replace an anonymous huge PMD entry with a page table mapping the same * folio at PTE granularity.
@@ -3281,53 +3317,14 @@ static void split_huge_pmd_to_ptes(struct vm_area_struct *vma,* Note that NUMA hinting access restrictions are not transferred to * avoid any possibility of altering permissions across VMAs. */ -if (state->freeze || - (!state->is_present && !state->is_device_private)) { -pte_t entry; -swp_entry_t swp_entry; - -for (i =0 , addr = haddr; i <HPAGE_PMD_NR ; i++, addr +=PAGE_SIZE ) { -if (state->write) - swp_entry =make_writable_migration_entry ( -page_to_pfn (page + i)); -else if (state->anon_exclusive) - swp_entry =make_readable_exclusive_migration_entry ( -page_to_pfn (page + i)); -else - swp_entry =make_readable_migration_entry ( -page_to_pfn (page + i)); -if (state->young) - swp_entry =make_migration_entry_young (swp_entry); -if (state->dirty) - swp_entry =make_migration_entry_dirty (swp_entry); - entry =swp_entry_to_pte (swp_entry); -if (state->soft_dirty) - entry =pte_swp_mksoft_dirty (entry); -if (state->uffd) - entry =pte_swp_mkuffd (entry); -VM_WARN_ON (!pte_none (ptep_get (pte + i))); -set_pte_at (mm, addr, pte + i, entry); - } - }else if (state->is_device_private) { -pte_t entry; -swp_entry_t swp_entry; +if (state->freeze || !state->is_present) { +for (i =0 , addr = haddr; i <HPAGE_PMD_NR ; + i++, addr +=PAGE_SIZE ) { +const unsigned long pfn =page_to_pfn (page + i); +const softleaf_t leaf = +split_pmd_make_softleaf (state, pfn); +pte_t entry =softleaf_to_pte (leaf); -for (i =0 , addr = haddr; i <HPAGE_PMD_NR ; i++, addr +=PAGE_SIZE ) { -/* -* anon_exclusive was already propagated to the relevant -* pages corresponding to the pte entries when freeze -* is false. -*/ -if (state->write) - swp_entry =make_writable_device_private_entry ( -page_to_pfn (page + i)); -else - swp_entry =make_readable_device_private_entry ( -page_to_pfn (page + i)); -/* -* Young and dirty bits are not progated via swp_entry -*/ - entry =swp_entry_to_pte (swp_entry);if (state->soft_dirty) entry =pte_swp_mksoft_dirty (entry);if (state->uffd)
Signature · 1 line
2.34.1
David Hildenbrand (Arm)
maintainerQuoted 10 lines · Yin Tirui
__split_huge_pmd_locked()currently decides from the VMA --vma_is_anonymous(), thenvma_is_special_huge()-- before inspecting the PMD entry, so split and zap can classify the same PMD differently. The entry itself should decide. The function also mixes the present and non-present cases, resulting in duplicated code.This series uses the PMD entry and its folio to decide how to handle each PMD, as
zap_huge_pmd()does. It separates the present and non-present paths and moves their common work into helper functions. After this change,__split_huge_pmd_locked()only selects the appropriate helper.
I skimmed over some of the patches and there seems to be valuable cleanups in there. As the MM list is right now extremely busy, I won't be able to review this right now, so unfortunately this will have to wait.
Signature · 3 lines
Cheers, David