mm/huge_memory: refactor zap_huge_pmd()
Subsystems (3)
Lorenzo Stoakes (Oracle)
reviewercover mm/huge_memory: refactor zap_huge_pmd()
The zap_huge_pmd() function is overly complicated, clean it up and also add an assert in the case that we encounter a buggy PMD entry that doesn't match expectations.
This is motivated by a bug discovered [0] where the PMD entry was none of:
- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entry
In zap_huge_pmd(), but due to the bug we manged to reach this code.
It is useful to explicitly call this out rather than have an arbitrary NULL pointer dereference happen, which also improves understanding of what's going on.
[0]:https://lore.kernel.org/all/[email protected]/
Lorenzo Stoakes (Oracle) (8):
mm/huge_memory: simplify vma_is_specal_huge()
mm/huge: avoid big else branch in zap_huge_pmd() mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc mm/huge_memory: handle buggy PMD entry in zap_huge_pmd() mm/huge_memory: add a common exit path to zap_huge_pmd() mm/huge_memory: remove unnecessary VM_BUG_ON_PAGE() mm/huge_memory: deduplicate zap deposited table call mm/huge_memory: deduplicate zap_huge_pmd() further by tracking state
include/linux/huge_mm.h | 8 +-- include/linux/mm.h | 16 ----- mm/huge_memory.c | 145 +++++++++++++++++++++++----------------- 3 files changed, 89 insertions(+), 80 deletions(-)
--
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 1/8 mm/huge_memory: simplify vma_is_specal_huge()
This function is confused - it overloads the term 'special' yet again, checks for DAX but in many cases the code explicitly excludes DAX before invoking the predicate.
It also unnecessarily checks for vma->vm_file - this has to be present for a driver to have set VMA_MIXEDMAP_BIT or VMA_PFNMAP_BIT.
In fact, a far simpler form of this is to reverse the DAX predicate and return false if DAX is set.
This makes sense from the point of view of 'special' as in vm_normal_page(), as DAX actually does potentially have retrievable folios.
Also there's no need to have this in mm.h so move it to huge_memory.c.
--- include/linux/huge_mm.h | 4 ++-- include/linux/mm.h | 16 ---------------- mm/huge_memory.c | 30 +++++++++++++++++++++++------- 3 files changed, 25 insertions(+), 25 deletions(-)
include/linux/huge_mm.h +2 -2
@@ -83,7 +83,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;* file is never split and the MAX_PAGECACHE_ORDER limit does not apply to * it. Same to PFNMAPs where there's neither page* nor pagecache. */ -#define THP_ORDERS_ALL_SPECIAL \ +#define THP_ORDERS_ALL_SPECIAL_DAX \ (BIT(PMD_ORDER) | BIT(PUD_ORDER))#define THP_ORDERS_ALL_FILE_DEFAULT \ ((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))
@@ -92,7 +92,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;* Mask of all large folio orders supported for THP. */ #define THP_ORDERS_ALL \ - (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT) + (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT)enum tva_type {TVA_SMAPS ,/* Exposing "THPeligible:" in smaps. */
include/linux/mm.h +0 -16
@@ -5077,22 +5077,6 @@ long copy_folio_from_user(struct folio *dst_folio,const void __user *usr_src, bool allow_pagefault); -/** -* vma_is_special_huge - Are transhuge page-table entries considered special? -* @vma: Pointer to the struct vm_area_struct to consider -* -* Whether transhuge page-table entries are considered "special" following -* the definition in vm_normal_page(). -* -* Return: true if transhuge page-table entries should be considered special, -* false otherwise. -*/ -static inline bool vma_is_special_huge (const struct vm_area_struct *vma) -{ -return vma_is_dax (vma) || (vma->vm_file && - (vma->vm_flags & (VM_PFNMAP |VM_MIXEDMAP ))); -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ #if MAX_NUMNODES >1
mm/huge_memory.c +23 -7
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)return !inode_is_open_for_write (inode) &&S_ISREG (inode->i_mode); } +/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge (struct vm_area_struct *vma) +{ +if (vma_is_dax (vma)) +return false; +return vma_test_any (vma,VMA_PFNMAP_BIT ,VMA_MIXEDMAP_BIT ); +} + unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type,
@@ -113,8 +121,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,/* Check the intersection of requested and supported orders. */ if (vma_is_anonymous (vma)) supported_orders =THP_ORDERS_ALL_ANON ; -else if (vma_is_special_huge (vma)) - supported_orders =THP_ORDERS_ALL_SPECIAL ; +else if (vma_is_dax (vma) ||vma_is_special_huge (vma)) + supported_orders =THP_ORDERS_ALL_SPECIAL_DAX ;else supported_orders =THP_ORDERS_ALL_FILE_DEFAULT ;
@@ -2431,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb->fullmm);arch_check_zapped_pmd (vma, orig_pmd);tlb_remove_pmd_tlb_entry (tlb, pmd, addr); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) { +if (vma_is_special_huge (vma)) {if (arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd);spin_unlock (ptl);
@@ -2933,7 +2941,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, orig_pud =pudp_huge_get_and_clear_full (vma, addr, pud, tlb->fullmm);arch_check_zapped_pud (vma, orig_pud);tlb_remove_pud_tlb_entry (tlb, pud, addr); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) { +if (vma_is_special_huge (vma)) {spin_unlock (ptl);/* No zero page support yet */ }else {
@@ -3084,7 +3092,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,*/ if (arch_needs_pgtable_deposit ())zap_deposited_table (mm, pmd); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) +if (vma_is_special_huge (vma))return ;if (unlikely (pmd_is_migration_entry (old_pmd))) {const softleaf_t old_entry =softleaf_from_pmd (old_pmd);
@@ -4645,8 +4653,16 @@ static void split_huge_pages_all(void)static inline bool vma_not_suitable_for_thp_split (struct vm_area_struct *vma) { -return vma_is_special_huge (vma) || (vma->vm_flags &VM_IO ) || -is_vm_hugetlb_page (vma); +if (vma_is_dax (vma)) +return true; +if (vma_is_special_huge (vma)) +return true; +if (vma_test (vma,VMA_IO_BIT )) +return true; +if (is_vm_hugetlb_page (vma)) +return true; + +return false; } static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
Signature · 1 line
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 2/8 mm/huge: avoid big else branch in zap_huge_pmd()
In reply to Lorenzo Stoakes (Oracle)We don't need to have an extra level of indentation, we can simply exit early in the first two branches.
--- mm/huge_memory.c | 87 +++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 42 deletions(-)
mm/huge_memory.c +45 -42
@@ -2421,8 +2421,10 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)int zap_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma,pmd_t *pmd,unsigned long addr) { -pmd_t orig_pmd; +struct folio *folio =NULL ; +int flush_needed =1 ;spinlock_t *ptl; +pmd_t orig_pmd;tlb_change_page_size (tlb,HPAGE_PMD_SIZE );
@@ -2443,59 +2445,60 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,if (arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd);spin_unlock (ptl); - }else if (is_huge_zero_pmd (orig_pmd)) { +return 1 ; + } +if (is_huge_zero_pmd (orig_pmd)) {if (!vma_is_dax (vma) ||arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd);spin_unlock (ptl); - }else { -struct folio *folio =NULL ; -int flush_needed =1 ; +return 1 ; + } -if (pmd_present (orig_pmd)) { -struct page *page =pmd_page (orig_pmd); +if (pmd_present (orig_pmd)) { +struct page *page =pmd_page (orig_pmd); - folio =page_folio (page); -folio_remove_rmap_pmd (folio, page, vma); -WARN_ON_ONCE (folio_mapcount (folio) <0 ); -VM_BUG_ON_PAGE (!PageHead (page), page); - }else if (pmd_is_valid_softleaf (orig_pmd)) { -const softleaf_t entry =softleaf_from_pmd (orig_pmd); + folio =page_folio (page); +folio_remove_rmap_pmd (folio, page, vma); +WARN_ON_ONCE (folio_mapcount (folio) <0 ); +VM_BUG_ON_PAGE (!PageHead (page), page); + }else if (pmd_is_valid_softleaf (orig_pmd)) { +const softleaf_t entry =softleaf_from_pmd (orig_pmd); - folio =softleaf_to_folio (entry); - flush_needed =0 ; + folio =softleaf_to_folio (entry); + flush_needed =0 ; -if (!thp_migration_supported ()) -WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" ); - } +if (!thp_migration_supported ()) +WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" ); + } -if (folio_test_anon (folio)) { +if (folio_test_anon (folio)) { +zap_deposited_table (tlb->mm, pmd); +add_mm_counter (tlb->mm,MM_ANONPAGES , -HPAGE_PMD_NR ); + }else { +if (arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd); -add_mm_counter (tlb->mm,MM_ANONPAGES , -HPAGE_PMD_NR ); - }else { -if (arch_needs_pgtable_deposit ()) -zap_deposited_table (tlb->mm, pmd); -add_mm_counter (tlb->mm,mm_counter_file (folio), - -HPAGE_PMD_NR ); - -/* -* Use flush_needed to indicate whether the PMD entry -* is present, instead of checking pmd_present() again. -*/ -if (flush_needed &&pmd_young (orig_pmd) && -likely (vma_has_recency (vma))) -folio_mark_accessed (folio); - } +add_mm_counter (tlb->mm,mm_counter_file (folio), + -HPAGE_PMD_NR ); -if (folio_is_device_private (folio)) { -folio_remove_rmap_pmd (folio, &folio->page, vma); -WARN_ON_ONCE (folio_mapcount (folio) <0 ); -folio_put (folio); - } +/* +* Use flush_needed to indicate whether the PMD entry +* is present, instead of checking pmd_present() again. +*/ +if (flush_needed &&pmd_young (orig_pmd) && +likely (vma_has_recency (vma))) +folio_mark_accessed (folio); + } -spin_unlock (ptl); -if (flush_needed) -tlb_remove_page_size (tlb, &folio->page,HPAGE_PMD_SIZE ); +if (folio_is_device_private (folio)) { +folio_remove_rmap_pmd (folio, &folio->page, vma); +WARN_ON_ONCE (folio_mapcount (folio) <0 ); +folio_put (folio); } + +spin_unlock (ptl); +if (flush_needed) +tlb_remove_page_size (tlb, &folio->page,HPAGE_PMD_SIZE ); +return 1 ; }
Signature · 1 line
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 3/8 mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc
In reply to Lorenzo Stoakes (Oracle)There's no need to use the ancient approach of returning an integer here, just return a boolean.
Also update flush_needed to be a boolean, similarly.
Also add a kdoc comment describing the function.
--- include/linux/huge_mm.h | 4 ++-- mm/huge_memory.c | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-)
include/linux/huge_mm.h +2 -2
@@ -27,8 +27,8 @@ static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)vm_fault_t do_huge_pmd_wp_page (struct vm_fault *vmf);bool madvise_free_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma,pmd_t *pmd,unsigned long addr,unsigned long next); -int zap_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma,pmd_t *pmd, -unsigned long addr); +bool zap_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma,pmd_t *pmd, +unsigned long addr);int zap_huge_pud (struct mmu_gather *tlb,struct vm_area_struct *vma,pud_t *pud,unsigned long addr); bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
mm/huge_memory.c +16 -7
@@ -2418,11 +2418,20 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)mm_dec_nr_ptes (mm); } -int zap_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma, +/** +* zap_huge_pmd - Zap a huge THP which is of PMD size. +* @tlb: The MMU gather TLB state associated with the operation. +* @vma: The VMA containing the range to zap. +* @pmd: A pointer to the leaf PMD entry. +* @addr: The virtual address for the range to zap. +* +* Returns: %true on success, %false otherwise. +*/ +bool zap_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma,pmd_t *pmd,unsigned long addr) {struct folio *folio =NULL ; -int flush_needed =1 ; +bool flush_needed = true;spinlock_t *ptl;pmd_t orig_pmd;
@@ -2430,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, ptl =__pmd_trans_huge_lock (pmd, vma);if (!ptl) -return 0 ; +return false;/* * For architectures like ppc64 we look at deposited pgtable * when calling pmdp_huge_get_and_clear. So do the
@@ -2445,13 +2454,13 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,if (arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd);spin_unlock (ptl); -return 1 ; +return true; }if (is_huge_zero_pmd (orig_pmd)) {if (!vma_is_dax (vma) ||arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd);spin_unlock (ptl); -return 1 ; +return true; }if (pmd_present (orig_pmd)) {
@@ -2465,7 +2474,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,const softleaf_t entry =softleaf_from_pmd (orig_pmd); folio =softleaf_to_folio (entry); - flush_needed =0 ; + flush_needed = false;if (!thp_migration_supported ())WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" );
@@ -2499,7 +2508,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,if (flush_needed)tlb_remove_page_size (tlb, &folio->page,HPAGE_PMD_SIZE ); -return 1 ; +return true; }#ifndef pmd_move_must_withdraw
Signature · 1 line
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 4/8 mm/huge_memory: handle buggy PMD entry in zap_huge_pmd()
In reply to Lorenzo Stoakes (Oracle)A recent bug I analysed [0] managed to, through a bug in the userfaultfd implementation, reach an invalid point in the zap_huge_pmd() code where the PMD was none of:
- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entry
The code at this point calls folio_test_anon() on a known-NULL folio. Having logic like this explicitly NULL dereference in the code is hard to understand, and makes debugging potentially more difficult.
Add an else branch to handle this case and WARN() and exit indicating failure.
[0]:https://lore.kernel.org/all/[email protected]/
--- mm/huge_memory.c | 4 ++++ 1 file changed, 4 insertions(+)
mm/huge_memory.c +4 -0
@@ -2478,6 +2478,10 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,if (!thp_migration_supported ())WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" ); + }else { +WARN_ON_ONCE (true); +spin_unlock (ptl); +return false; }if (folio_test_anon (folio)) {
Signature · 1 line
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 5/8 mm/huge_memory: add a common exit path to zap_huge_pmd()
In reply to Lorenzo Stoakes (Oracle)Other than when we acquire the PTL, we always need to unlock the PTL, and optionally need to flush on exit.
The code is currently very duplicated in this respect, so default flush_needed to false, set it true in the case in which it's required, then share the same logic for all exit paths.
This also makes flush_needed make more sense as a function-scope value (we don't need to flush for the PFN map/mixed map, zero huge, error cases for instance).
--- mm/huge_memory.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-)
mm/huge_memory.c +9 -10
@@ -2431,7 +2431,8 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr) {struct folio *folio =NULL ; -bool flush_needed = true; +bool flush_needed = false; +bool ret = true;spinlock_t *ptl;pmd_t orig_pmd;
@@ -2453,19 +2454,18 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,if (vma_is_special_huge (vma)) {if (arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd); -spin_unlock (ptl); -return true; + goto out; }if (is_huge_zero_pmd (orig_pmd)) {if (!vma_is_dax (vma) ||arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd); -spin_unlock (ptl); -return true; + goto out; }if (pmd_present (orig_pmd)) {struct page *page =pmd_page (orig_pmd); + flush_needed = true; folio =page_folio (page);folio_remove_rmap_pmd (folio, page, vma);WARN_ON_ONCE (folio_mapcount (folio) <0 );
@@ -2474,14 +2474,13 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,const softleaf_t entry =softleaf_from_pmd (orig_pmd); folio =softleaf_to_folio (entry); - flush_needed = false;if (!thp_migration_supported ())WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" ); }else {WARN_ON_ONCE (true); -spin_unlock (ptl); -return false; + ret = false; + goto out; }if (folio_test_anon (folio)) {
@@ -2508,11 +2507,11 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,folio_put (folio); } +out:spin_unlock (ptl);if (flush_needed)tlb_remove_page_size (tlb, &folio->page,HPAGE_PMD_SIZE ); - -return true; +return ret; }#ifndef pmd_move_must_withdraw
Signature · 1 line
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 6/8 mm/huge_memory: remove unnecessary VM_BUG_ON_PAGE()
In reply to Lorenzo Stoakes (Oracle)This has been around since the beginnings of the THP implementation. I think we can safely assume that, if we have a THP folio, it will have a head page.
--- mm/huge_memory.c | 1 - 1 file changed, 1 deletion(-)
mm/huge_memory.c +0 -1
@@ -2469,7 +2469,6 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, folio =page_folio (page);folio_remove_rmap_pmd (folio, page, vma);WARN_ON_ONCE (folio_mapcount (folio) <0 ); -VM_BUG_ON_PAGE (!PageHead (page), page); }else if (pmd_is_valid_softleaf (orig_pmd)) {const softleaf_t entry =softleaf_from_pmd (orig_pmd);
Signature · 1 line
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 7/8 mm/huge_memory: deduplicate zap deposited table call
In reply to Lorenzo Stoakes (Oracle)Rather than having separate logic for each case determining whether to zap the deposited table, simply track this via a boolean.
We check separately if the architecture requires it.
Also use pmd_folio() direct in the present case.
--- mm/huge_memory.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-)
mm/huge_memory.c +9 -12
@@ -2432,6 +2432,7 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, {struct folio *folio =NULL ;bool flush_needed = false; +bool needs_deposit = false;bool ret = true;spinlock_t *ptl;pmd_t orig_pmd;
@@ -2451,23 +2452,18 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb->fullmm);arch_check_zapped_pmd (vma, orig_pmd);tlb_remove_pmd_tlb_entry (tlb, pmd, addr); -if (vma_is_special_huge (vma)) { -if (arch_needs_pgtable_deposit ()) -zap_deposited_table (tlb->mm, pmd); +if (vma_is_special_huge (vma)) goto out; - }if (is_huge_zero_pmd (orig_pmd)) { -if (!vma_is_dax (vma) ||arch_needs_pgtable_deposit ()) -zap_deposited_table (tlb->mm, pmd); + needs_deposit = !vma_is_dax (vma); goto out; }if (pmd_present (orig_pmd)) { -struct page *page =pmd_page (orig_pmd); + folio =pmd_folio (orig_pmd); flush_needed = true; - folio =page_folio (page); -folio_remove_rmap_pmd (folio, page, vma); +folio_remove_rmap_pmd (folio, &folio->page, vma);WARN_ON_ONCE (folio_mapcount (folio) <0 ); }else if (pmd_is_valid_softleaf (orig_pmd)) {const softleaf_t entry =softleaf_from_pmd (orig_pmd);
@@ -2483,11 +2479,9 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, }if (folio_test_anon (folio)) { -zap_deposited_table (tlb->mm, pmd); + needs_deposit = true;add_mm_counter (tlb->mm,MM_ANONPAGES , -HPAGE_PMD_NR ); }else { -if (arch_needs_pgtable_deposit ()) -zap_deposited_table (tlb->mm, pmd);add_mm_counter (tlb->mm,mm_counter_file (folio), -HPAGE_PMD_NR );
@@ -2507,6 +2501,9 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, } out: +if (arch_needs_pgtable_deposit () || needs_deposit) +zap_deposited_table (tlb->mm, pmd); +spin_unlock (ptl);if (flush_needed)tlb_remove_page_size (tlb, &folio->page,HPAGE_PMD_SIZE );
--
2.53.0
Lorenzo Stoakes (Oracle)
reviewerPATCH 8/8 mm/huge_memory: deduplicate zap_huge_pmd() further by tracking state
In reply to Lorenzo Stoakes (Oracle)The flush_needed boolean is really tracking whether a PMD entry is present, so use it that way directly and rename it to is_present.
Deduplicate the folio_remove_rmap_pmd() and folio map count warning between present and device private by tracking where we need to remove the rmap.
We can also remove the comment about using flush_needed to track whether a PMD entry is present as it's now irrelevant.
--- mm/huge_memory.c | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-)
mm/huge_memory.c +13 -15
@@ -2430,9 +2430,10 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)bool zap_huge_pmd (struct mmu_gather *tlb,struct vm_area_struct *vma,pmd_t *pmd,unsigned long addr) { +bool needs_remove_rmap = false;struct folio *folio =NULL ; -bool flush_needed = false;bool needs_deposit = false; +bool is_present = false;bool ret = true;spinlock_t *ptl;pmd_t orig_pmd;
@@ -2450,6 +2451,7 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,*/ orig_pmd =pmdp_huge_get_and_clear_full (vma, addr, pmd, tlb->fullmm); +arch_check_zapped_pmd (vma, orig_pmd);tlb_remove_pmd_tlb_entry (tlb, pmd, addr);if (vma_is_special_huge (vma))
@@ -2459,17 +2461,15 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto out; } -if (pmd_present (orig_pmd)) { + is_present =pmd_present (orig_pmd); +if (is_present) { folio =pmd_folio (orig_pmd); - - flush_needed = true; -folio_remove_rmap_pmd (folio, &folio->page, vma); -WARN_ON_ONCE (folio_mapcount (folio) <0 ); + needs_remove_rmap = true; }else if (pmd_is_valid_softleaf (orig_pmd)) {const softleaf_t entry =softleaf_from_pmd (orig_pmd); folio =softleaf_to_folio (entry); - + needs_remove_rmap =folio_is_device_private (folio);if (!thp_migration_supported ())WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" ); }else {
@@ -2485,27 +2485,25 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,add_mm_counter (tlb->mm,mm_counter_file (folio), -HPAGE_PMD_NR ); -/* -* Use flush_needed to indicate whether the PMD entry -* is present, instead of checking pmd_present() again. -*/ -if (flush_needed &&pmd_young (orig_pmd) && +if (is_present &&pmd_young (orig_pmd) &&likely (vma_has_recency (vma)))folio_mark_accessed (folio); } -if (folio_is_device_private (folio)) { +if (needs_remove_rmap) {folio_remove_rmap_pmd (folio, &folio->page, vma);WARN_ON_ONCE (folio_mapcount (folio) <0 ); -folio_put (folio); } out:if (arch_needs_pgtable_deposit () || needs_deposit)zap_deposited_table (tlb->mm, pmd); +if (needs_remove_rmap && !is_present) +folio_put (folio); +spin_unlock (ptl); -if (flush_needed) +if (is_present)tlb_remove_page_size (tlb, &folio->page,HPAGE_PMD_SIZE );return ret; }
Signature · 1 line
2.53.0
David Hildenbrand (Arm)
maintainerRe: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()
In reply to Lorenzo Stoakes (Oracle)mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109
Quoted 12 lines
+++
b/mm/huge_memory.cmm/huge_memory.c +7
-0@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)return !inode_is_open_for_write (inode) &&S_ISREG (inode->i_mode); } +/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge (struct vm_area_struct *vma) +{ +if (vma_is_dax (vma)) +return false; +return vma_test_any (vma,VMA_PFNMAP_BIT ,VMA_MIXEDMAP_BIT ); +}
I was hoping that we could make this whole code look more like PTE code by using vm_normal_page_pmd() ... :)
Signature · 3 lines
Cheers, David
Re: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()
In reply to Lorenzo Stoakes (Oracle)Quoted 27 lines · Lorenzo Stoakes (Oracle)
This function is confused - it overloads the term 'special' yet again, checks for DAX but in many cases the code explicitly excludes DAX before invoking the predicate.
It also unnecessarily checks for
vma->vm_file- this has to be present for a driver to have setVMA_MIXEDMAP_BITorVMA_PFNMAP_BIT.In fact, a far simpler form of this is to reverse the DAX predicate and return false if DAX is set.
This makes sense from the point of view of 'special' as in
vm_normal_page(), as DAX actually does potentially have retrievable folios.Also there's no need to have this in
mm.hso move it tohuge_memory.c.Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>--- include/linux/huge_mm.h | 4 ++-- include/linux/mm.h | 16 ---------------- mm/huge_memory.c | 30 +++++++++++++++++++++++------- 3 files changed, 25 insertions(+), 25 deletions(-)include/linux/huge_mm.h +0
-0
include/linux/huge_mm.h extern struct kobj_attribute thpsize_shmem_enabled_attr; · line 86
Quoted 5 lines
include/linux/huge_mm.h +1
-1@@ -83,7 +83,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;* file is never split and the MAX_PAGECACHE_ORDER limit does not apply to * it. Same to PFNMAPs where there's neither page* nor pagecache. */ -#define THP_ORDERS_ALL_SPECIAL \ +#define THP_ORDERS_ALL_SPECIAL_DAX \
As mentioned in the comments, the pfnmap case is also include in the 'special' case, right?
Quoted 115 lines
include/linux/huge_mm.h +1
-1(BIT(PMD_ORDER) | BIT(PUD_ORDER))#define THP_ORDERS_ALL_FILE_DEFAULT \ ((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))@@ -92,7 +92,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;* Mask of all large folio orders supported for THP. */ #define THP_ORDERS_ALL \ - (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT) + (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT)enum tva_type {TVA_SMAPS ,/* Exposing "THPeligible:" in smaps. */ include/linux/mm.h +0
-16@@ -5077,22 +5077,6 @@ long copy_folio_from_user(struct folio *dst_folio,const void __user *usr_src, bool allow_pagefault); -/** -* vma_is_special_huge - Are transhuge page-table entries considered special? -* @vma: Pointer to the struct vm_area_struct to consider -* -* Whether transhuge page-table entries are considered "special" following -* the definition in vm_normal_page(). -* -* Return: true if transhuge page-table entries should be considered special, -* false otherwise. -*/ -static inline bool vma_is_special_huge (const struct vm_area_struct *vma) -{ -return vma_is_dax (vma) || (vma->vm_file && - (vma->vm_flags & (VM_PFNMAP |VM_MIXEDMAP ))); -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ #if MAX_NUMNODES >1 mm/huge_memory.c +23
-7@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)return !inode_is_open_for_write (inode) &&S_ISREG (inode->i_mode); } +/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge (struct vm_area_struct *vma) +{ +if (vma_is_dax (vma)) +return false; +return vma_test_any (vma,VMA_PFNMAP_BIT ,VMA_MIXEDMAP_BIT ); +} + unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type,@@ -113,8 +121,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,/* Check the intersection of requested and supported orders. */ if (vma_is_anonymous (vma)) supported_orders =THP_ORDERS_ALL_ANON ; -else if (vma_is_special_huge (vma)) - supported_orders =THP_ORDERS_ALL_SPECIAL ; +else if (vma_is_dax (vma) ||vma_is_special_huge (vma)) + supported_orders =THP_ORDERS_ALL_SPECIAL_DAX ;else supported_orders =THP_ORDERS_ALL_FILE_DEFAULT ;@@ -2431,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb->fullmm);arch_check_zapped_pmd (vma, orig_pmd);tlb_remove_pmd_tlb_entry (tlb, pmd, addr); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) { +if (vma_is_special_huge (vma)) {if (arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd);spin_unlock (ptl);@@ -2933,7 +2941,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, orig_pud =pudp_huge_get_and_clear_full (vma, addr, pud, tlb->fullmm);arch_check_zapped_pud (vma, orig_pud);tlb_remove_pud_tlb_entry (tlb, pud, addr); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) { +if (vma_is_special_huge (vma)) {spin_unlock (ptl);/* No zero page support yet */ }else {@@ -3084,7 +3092,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,*/ if (arch_needs_pgtable_deposit ())zap_deposited_table (mm, pmd); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) +if (vma_is_special_huge (vma))return ;if (unlikely (pmd_is_migration_entry (old_pmd))) {const softleaf_t old_entry =softleaf_from_pmd (old_pmd);@@ -4645,8 +4653,16 @@ static void split_huge_pages_all(void)static inline bool vma_not_suitable_for_thp_split (struct vm_area_struct *vma) { -return vma_is_special_huge (vma) || (vma->vm_flags &VM_IO ) || -is_vm_hugetlb_page (vma); +if (vma_is_dax (vma)) +return true; +if (vma_is_special_huge (vma)) +return true; +if (vma_test (vma,VMA_IO_BIT )) +return true; +if (is_vm_hugetlb_page (vma)) +return true; + +return false; } static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
Re: [PATCH 2/8] mm/huge: avoid big else branch in zap_huge_pmd()
In reply to Lorenzo Stoakes (Oracle)Quoted 8 lines · Lorenzo Stoakes (Oracle)
We don't need to have an extra level of indentation, we can simply exit early in the first two branches.
Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>--- mm/huge_memory.c | 87 +++++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 42 deletions(-)
The code here isn't too complex, otherwise we can introduce the function like do_zap_pte_range().
Re: [PATCH 3/8] mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc
In reply to Lorenzo Stoakes (Oracle)Quoted 5 lines · Lorenzo Stoakes (Oracle)
There's no need to use the ancient approach of returning an integer here, just return a boolean.
Also update
flush_neededto be a boolean, similarly.
Quoted 8 lines
Also add a kdoc comment describing the function.
Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>--- include/linux/huge_mm.h | 4 ++-- mm/huge_memory.c | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-)
Re: [PATCH 2/8] mm/huge: avoid big else branch in zap_huge_pmd()
In reply to Lorenzo Stoakes (Oracle)Quoted 5 lines · Lorenzo Stoakes (Oracle)
We don't need to have an extra level of indentation, we can simply exit early in the first two branches.
Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>---
LGTM.
Re: [PATCH 3/8] mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc
In reply to Lorenzo Stoakes (Oracle)Quoted 9 lines · Lorenzo Stoakes (Oracle)
There's no need to use the ancient approach of returning an integer here, just return a boolean.
Also update
flush_neededto be a boolean, similarly.Also add a kdoc comment describing the function.
Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>---
Make sense to me.
Baolin Wang
reviewerRe: [PATCH 4/8] mm/huge_memory: handle buggy PMD entry in zap_huge_pmd()
In reply to Lorenzo Stoakes (Oracle)Quoted 28 lines · Lorenzo Stoakes (Oracle)
A recent bug I analysed [0] managed to, through a bug in the userfaultfd implementation, reach an invalid point in the
zap_huge_pmd()code where the PMD was none of:- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entryThe code at this point calls
folio_test_anon()on a known-NULL folio. Having logic like this explicitly NULL dereference in the code is hard to understand, and makes debugging potentially more difficult.Add an else branch to handle this case and
WARN()and exit indicating failure.[0]:https://lore.kernel.org/all/[email protected]/
Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>--- mm/huge_memory.c | 4 ++++ 1 file changed, 4 insertions(+)mm/huge_memory.c +0
-0
mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2483
Quoted 6 lines
mm/huge_memory.c +3
-0@@ -2478,6 +2478,10 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,if (!thp_migration_supported ())WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" ); + }else { +WARN_ON_ONCE (true); +spin_unlock (ptl);
The warning looks reasonable to me, but ...
mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2484
Quoted 1 line
mm/huge_memory.c +1
-0+return false;
IIUC, if we return false here, the caller zap_pmd_range() will fall back to call zap_pte_range(). Since pmd_trans_huge(pmd) returns true, zap_pte_range() will simply return 'addr', causing an infinite loop in zap_pmd_range(), right?
Re: [PATCH 6/8] mm/huge_memory: remove unnecessary VM_BUG_ON_PAGE()
In reply to Lorenzo Stoakes (Oracle)Quoted 6 lines · Lorenzo Stoakes (Oracle)
This has been around since the beginnings of the THP implementation. I think we can safely assume that, if we have a THP folio, it will have a head page.
Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>---
Yes, it is under the PTL, so the PMD entry should be stable. I also can't think of any case where it can be not a head page.
Lorenzo Stoakes (Oracle)
reviewerRe: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()
In reply to David Hildenbrand (Arm)Quoted 5 lines · David Hildenbrand (Arm)
Quoted 5 lines
+++
b/mm/huge_memory.cmm/huge_memory.c +0
-0@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)return !inode_is_open_for_write (inode) &&S_ISREG (inode->i_mode); }
Quoted 10 lines
mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109
Quoted 7 lines
mm/huge_memory.c +7
-0+/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge (struct vm_area_struct *vma) +{ +if (vma_is_dax (vma)) +return false; +return vma_test_any (vma,VMA_PFNMAP_BIT ,VMA_MIXEDMAP_BIT ); +}I was hoping that we could make this whole code look more like PTE code by using
vm_normal_page_pmd()... :)
Could that be a follow up?
I did want to do something where we get the folio and use that to figure things out, but there's a lot of special cases esp. around DAX that make that tricky.
Quoted 5 lines
--
Cheers,David
Thanks, Lorenzo
Lorenzo Stoakes (Oracle)
reviewerRe: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()
In reply to Qi ZhengQuoted 25 lines · Qi Zheng
Quoted 22 lines · Lorenzo Stoakes (Oracle)
This function is confused - it overloads the term 'special' yet again, checks for DAX but in many cases the code explicitly excludes DAX before invoking the predicate.
It also unnecessarily checks for
vma->vm_file- this has to be present for a driver to have setVMA_MIXEDMAP_BITorVMA_PFNMAP_BIT.In fact, a far simpler form of this is to reverse the DAX predicate and return false if DAX is set.
This makes sense from the point of view of 'special' as in
vm_normal_page(), as DAX actually does potentially have retrievable folios.Also there's no need to have this in
mm.hso move it tohuge_memory.c.Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>--- include/linux/huge_mm.h | 4 ++-- include/linux/mm.h | 16 ---------------- mm/huge_memory.c | 30 +++++++++++++++++++++++------- 3 files changed, 25 insertions(+), 25 deletions(-)
Quoted 13 lines
include/linux/huge_mm.h extern struct kobj_attribute thpsize_shmem_enabled_attr; · line 86
Quoted 10 lines
include/linux/huge_mm.h +1
-1@@ -83,7 +83,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;* file is never split and the MAX_PAGECACHE_ORDER limit does not apply to * it. Same to PFNMAPs where there's neither page* nor pagecache. */ -#define THP_ORDERS_ALL_SPECIAL \ +#define THP_ORDERS_ALL_SPECIAL_DAX \As mentioned in the comments, the pfnmap case is also include in the 'special' case, right?
Yeah special = pfnmap, mixedmap. so renaming to SPECIAL_DAX to make clear it's either dax or 'special' in the meaning of vm_normal_page().
Quoted 110 lines
Quoted 109 lines
include/linux/huge_mm.h +1
-1(BIT(PMD_ORDER) | BIT(PUD_ORDER))#define THP_ORDERS_ALL_FILE_DEFAULT \ ((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))@@ -92,7 +92,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;* Mask of all large folio orders supported for THP. */ #define THP_ORDERS_ALL \ - (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT) + (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT)enum tva_type {TVA_SMAPS ,/* Exposing "THPeligible:" in smaps. */ include/linux/mm.h +0
-16@@ -5077,22 +5077,6 @@ long copy_folio_from_user(struct folio *dst_folio,const void __user *usr_src, bool allow_pagefault); -/** -* vma_is_special_huge - Are transhuge page-table entries considered special? -* @vma: Pointer to the struct vm_area_struct to consider -* -* Whether transhuge page-table entries are considered "special" following -* the definition in vm_normal_page(). -* -* Return: true if transhuge page-table entries should be considered special, -* false otherwise. -*/ -static inline bool vma_is_special_huge (const struct vm_area_struct *vma) -{ -return vma_is_dax (vma) || (vma->vm_file && - (vma->vm_flags & (VM_PFNMAP |VM_MIXEDMAP ))); -} -#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ #if MAX_NUMNODES >1 mm/huge_memory.c +23
-7@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)return !inode_is_open_for_write (inode) &&S_ISREG (inode->i_mode); } +/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge (struct vm_area_struct *vma) +{ +if (vma_is_dax (vma)) +return false; +return vma_test_any (vma,VMA_PFNMAP_BIT ,VMA_MIXEDMAP_BIT ); +} + unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type,@@ -113,8 +121,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,/* Check the intersection of requested and supported orders. */ if (vma_is_anonymous (vma)) supported_orders =THP_ORDERS_ALL_ANON ; -else if (vma_is_special_huge (vma)) - supported_orders =THP_ORDERS_ALL_SPECIAL ; +else if (vma_is_dax (vma) ||vma_is_special_huge (vma)) + supported_orders =THP_ORDERS_ALL_SPECIAL_DAX ;else supported_orders =THP_ORDERS_ALL_FILE_DEFAULT ;@@ -2431,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb->fullmm);arch_check_zapped_pmd (vma, orig_pmd);tlb_remove_pmd_tlb_entry (tlb, pmd, addr); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) { +if (vma_is_special_huge (vma)) {if (arch_needs_pgtable_deposit ())zap_deposited_table (tlb->mm, pmd);spin_unlock (ptl);@@ -2933,7 +2941,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, orig_pud =pudp_huge_get_and_clear_full (vma, addr, pud, tlb->fullmm);arch_check_zapped_pud (vma, orig_pud);tlb_remove_pud_tlb_entry (tlb, pud, addr); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) { +if (vma_is_special_huge (vma)) {spin_unlock (ptl);/* No zero page support yet */ }else {@@ -3084,7 +3092,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,*/ if (arch_needs_pgtable_deposit ())zap_deposited_table (mm, pmd); -if (!vma_is_dax (vma) &&vma_is_special_huge (vma)) +if (vma_is_special_huge (vma))return ;if (unlikely (pmd_is_migration_entry (old_pmd))) {const softleaf_t old_entry =softleaf_from_pmd (old_pmd);@@ -4645,8 +4653,16 @@ static void split_huge_pages_all(void)static inline bool vma_not_suitable_for_thp_split (struct vm_area_struct *vma) { -return vma_is_special_huge (vma) || (vma->vm_flags &VM_IO ) || -is_vm_hugetlb_page (vma); +if (vma_is_dax (vma)) +return true; +if (vma_is_special_huge (vma)) +return true; +if (vma_test (vma,VMA_IO_BIT )) +return true; +if (is_vm_hugetlb_page (vma)) +return true; + +return false; } static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
Lorenzo Stoakes (Oracle)
reviewerRe: [PATCH 4/8] mm/huge_memory: handle buggy PMD entry in zap_huge_pmd()
In reply to Baolin WangQuoted 39 lines · Baolin Wang
Quoted 28 lines · Lorenzo Stoakes (Oracle)
A recent bug I analysed [0] managed to, through a bug in the userfaultfd implementation, reach an invalid point in the
zap_huge_pmd()code where the PMD was none of:- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entryThe code at this point calls
folio_test_anon()on a known-NULL folio. Having logic like this explicitly NULL dereference in the code is hard to understand, and makes debugging potentially more difficult.Add an else branch to handle this case and
WARN()and exit indicating failure.[0]:https://lore.kernel.org/all/[email protected]/
Signed-off-by Lorenzo Stoakes (Oracle) <[email protected]>--- mm/huge_memory.c | 4 ++++ 1 file changed, 4 insertions(+)mm/huge_memory.c +0
-0mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2482
Quoted 5 lines
mm/huge_memory.c +3
-0@@ -2478,6 +2478,10 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,if (!thp_migration_supported ())WARN_ONCE (1 ,"Non present huge pmd without pmd migration enabled!" ); + }else { +WARN_ON_ONCE (true); +spin_unlock (ptl);The warning looks reasonable to me, but ...
Quoted 6 lines
mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2484
Quoted 1 line
mm/huge_memory.c +1
-0+return false;IIUC, if we return false here, the caller
zap_pmd_range()will fall back to callzap_pte_range(). Sincepmd_trans_huge(pmd)returns true,zap_pte_range()will simply return 'addr', causing an infinite loop inzap_pmd_range(), right?
You mean because:
start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) return addr;
In any case it looks like it degrades a false to potentially carrying on forever:
addr = zap_pte_range(tlb, vma, pmd, addr, next, details); if (addr != next) pmd--;
So yeah, this should be a true, annoyingly.
Will fix thanks!
Cheers, Lorenzo
David Hildenbrand (Arm)
maintainerRe: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()
In reply to Lorenzo Stoakes (Oracle)Quoted 17 lines · Lorenzo Stoakes (Oracle)
Quoted 5 lines · David Hildenbrand (Arm)
Quoted 5 lines
+++
b/mm/huge_memory.cmm/huge_memory.c +0
-0@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)return !inode_is_open_for_write (inode) &&S_ISREG (inode->i_mode); }Quoted 10 lines
mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109
Quoted 7 lines
mm/huge_memory.c +7
-0+/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge (struct vm_area_struct *vma) +{ +if (vma_is_dax (vma)) +return false; +return vma_test_any (vma,VMA_PFNMAP_BIT ,VMA_MIXEDMAP_BIT ); +}I was hoping that we could make this whole code look more like PTE code by using
vm_normal_page_pmd()... :)
Quoted 1 line
Could that be a follow up?
I guess so, but the code is so awful :(
Signature · 3 lines
Cheers, David
Lorenzo Stoakes (Oracle)
reviewerRe: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()
Quoted 18 lines · David Hildenbrand (Arm)
Quoted 17 lines · Lorenzo Stoakes (Oracle)
Quoted 15 lines · David Hildenbrand (Arm)
mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109
Quoted 12 lines
+++
b/mm/huge_memory.cmm/huge_memory.c +7
-0@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)return !inode_is_open_for_write (inode) &&S_ISREG (inode->i_mode); } +/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge (struct vm_area_struct *vma) +{ +if (vma_is_dax (vma)) +return false; +return vma_test_any (vma,VMA_PFNMAP_BIT ,VMA_MIXEDMAP_BIT ); +}I was hoping that we could make this whole code look more like PTE code by using
vm_normal_page_pmd()... :)
Quoted 3 lines
Quoted 1 line
Could that be a follow up?
I guess so, but the code is so awful :(
well I udated the code to use vm_normal_folio_pmd() :) at least should get us closer to common logic.
Cheers, Lorenzo