Skip to contentSkip to conversation

mm/huge_memory: refactor zap_huge_pmd()

PATCH Superseded Lorenzo Stoakes (Oracle) 4 participants 22 messages Mar 18 – Mar 19

Subsystems (3)

v1 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

Lorenzo Stoakes (Oracle)

author reviewer
#0 lore raw

cover mm/huge_memory: refactor zap_huge_pmd()

The zap_huge_pmd() function is overly complicated, clean it up and also add an assert in the case that we encounter a buggy PMD entry that doesn't match expectations.

This is motivated by a bug discovered [0] where the PMD entry was none of:

- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entry

In zap_huge_pmd(), but due to the bug we manged to reach this code.

It is useful to explicitly call this out rather than have an arbitrary NULL pointer dereference happen, which also improves understanding of what's going on.

[0]:https://lore.kernel.org/all/[email protected]/

Lorenzo Stoakes (Oracle) (8):
mm/huge_memory: simplify vma_is_specal_huge()
mm/huge: avoid big else branch in zap_huge_pmd() mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc mm/huge_memory: handle buggy PMD entry in zap_huge_pmd() mm/huge_memory: add a common exit path to zap_huge_pmd() mm/huge_memory: remove unnecessary VM_BUG_ON_PAGE() mm/huge_memory: deduplicate zap deposited table call mm/huge_memory: deduplicate zap_huge_pmd() further by tracking state

 include/linux/huge_mm.h |   8 +--
 include/linux/mm.h      |  16 -----
 mm/huge_memory.c        | 145 +++++++++++++++++++++++-----------------
 3 files changed, 89 insertions(+), 80 deletions(-)

--
2.53.0

author reviewer
#1 lore raw

PATCH 1/8 mm/huge_memory: simplify vma_is_specal_huge()

This function is confused - it overloads the term 'special' yet again, checks for DAX but in many cases the code explicitly excludes DAX before invoking the predicate.

It also unnecessarily checks for vma->vm_file - this has to be present for a driver to have set VMA_MIXEDMAP_BIT or VMA_PFNMAP_BIT.

In fact, a far simpler form of this is to reverse the DAX predicate and return false if DAX is set.

This makes sense from the point of view of 'special' as in vm_normal_page(), as DAX actually does potentially have retrievable folios.

Also there's no need to have this in mm.h so move it to huge_memory.c.

---
 include/linux/huge_mm.h |  4 ++--
 include/linux/mm.h      | 16 ----------------
 mm/huge_memory.c        | 30 +++++++++++++++++++++++-------
 3 files changed, 25 insertions(+), 25 deletions(-)
include/linux/huge_mm.h +2 -2
index bd7f0e1d8094..61fda1672b29 100644
@@ -83,7 +83,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;
  * file is never split and the MAX_PAGECACHE_ORDER limit does not apply to
  * it.  Same to PFNMAPs where there's neither page* nor pagecache.
  */
-#define THP_ORDERS_ALL_SPECIAL		\
+#define THP_ORDERS_ALL_SPECIAL_DAX	\
 	(BIT(PMD_ORDER) | BIT(PUD_ORDER))
 #define THP_ORDERS_ALL_FILE_DEFAULT	\
 	((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))
@@ -92,7 +92,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;
  * Mask of all large folio orders supported for THP.
  */
 #define THP_ORDERS_ALL	\
-	(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT)
+	(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT)
 
 enum tva_type {
 	TVA_SMAPS,		/* Exposing "THPeligible:" in smaps. */
include/linux/mm.h +0 -16
index 6f0a3edb24e1..50d68b092204 100644
@@ -5077,22 +5077,6 @@ long copy_folio_from_user(struct folio *dst_folio,
 			   const void __user *usr_src,
 			   bool allow_pagefault);
 
-/**
- * vma_is_special_huge - Are transhuge page-table entries considered special?
- * @vma: Pointer to the struct vm_area_struct to consider
- *
- * Whether transhuge page-table entries are considered "special" following
- * the definition in vm_normal_page().
- *
- * Return: true if transhuge page-table entries should be considered special,
- * false otherwise.
- */
-static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
-{
-	return vma_is_dax(vma) || (vma->vm_file &&
-				   (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
-}
-
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
 
 #if MAX_NUMNODES > 1
mm/huge_memory.c +23 -7
index 3fc02913b63e..f76edfa91e96 100644
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
 	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
 }
 
+/* If returns true, we are unable to access the VMA's folios. */
+static bool vma_is_special_huge(struct vm_area_struct *vma)
+{
+	if (vma_is_dax(vma))
+		return false;
+	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}
+
 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 					 vm_flags_t vm_flags,
 					 enum tva_type type,
@@ -113,8 +121,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 	/* Check the intersection of requested and supported orders. */
 	if (vma_is_anonymous(vma))
 		supported_orders = THP_ORDERS_ALL_ANON;
-	else if (vma_is_special_huge(vma))
-		supported_orders = THP_ORDERS_ALL_SPECIAL;
+	else if (vma_is_dax(vma) || vma_is_special_huge(vma))
+		supported_orders = THP_ORDERS_ALL_SPECIAL_DAX;
 	else
 		supported_orders = THP_ORDERS_ALL_FILE_DEFAULT;
 
@@ -2431,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 						tlb->fullmm);
 	arch_check_zapped_pmd(vma, orig_pmd);
 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
-	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
+	if (vma_is_special_huge(vma)) {
 		if (arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
 		spin_unlock(ptl);
@@ -2933,7 +2941,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm);
 	arch_check_zapped_pud(vma, orig_pud);
 	tlb_remove_pud_tlb_entry(tlb, pud, addr);
-	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
+	if (vma_is_special_huge(vma)) {
 		spin_unlock(ptl);
 		/* No zero page support yet */
 	} else {
@@ -3084,7 +3092,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
 		 */
 		if (arch_needs_pgtable_deposit())
 			zap_deposited_table(mm, pmd);
-		if (!vma_is_dax(vma) && vma_is_special_huge(vma))
+		if (vma_is_special_huge(vma))
 			return;
 		if (unlikely(pmd_is_migration_entry(old_pmd))) {
 			const softleaf_t old_entry = softleaf_from_pmd(old_pmd);
@@ -4645,8 +4653,16 @@ static void split_huge_pages_all(void)
 
 static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
 {
-	return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||
-		    is_vm_hugetlb_page(vma);
+	if (vma_is_dax(vma))
+		return true;
+	if (vma_is_special_huge(vma))
+		return true;
+	if (vma_test(vma, VMA_IO_BIT))
+		return true;
+	if (is_vm_hugetlb_page(vma))
+		return true;
+
+	return false;
 }
 
 static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
Signature · 1 line
2.53.0
author reviewer
#8 lore raw

PATCH 2/8 mm/huge: avoid big else branch in zap_huge_pmd()

In reply to Lorenzo Stoakes (Oracle)

We don't need to have an extra level of indentation, we can simply exit early in the first two branches.

---
 mm/huge_memory.c | 87 +++++++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 42 deletions(-)
mm/huge_memory.c +45 -42
index f76edfa91e96..4ebe1f19341e 100644
@@ -2421,8 +2421,10 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
 int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		 pmd_t *pmd, unsigned long addr)
 {
-	pmd_t orig_pmd;
+	struct folio *folio = NULL;
+	int flush_needed = 1;
 	spinlock_t *ptl;
+	pmd_t orig_pmd;
 
 	tlb_change_page_size(tlb, HPAGE_PMD_SIZE);
 
@@ -2443,59 +2445,60 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		if (arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
 		spin_unlock(ptl);
-	} else if (is_huge_zero_pmd(orig_pmd)) {
+		return 1;
+	}
+	if (is_huge_zero_pmd(orig_pmd)) {
 		if (!vma_is_dax(vma) || arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
 		spin_unlock(ptl);
-	} else {
-		struct folio *folio = NULL;
-		int flush_needed = 1;
+		return 1;
+	}
 
-		if (pmd_present(orig_pmd)) {
-			struct page *page = pmd_page(orig_pmd);
+	if (pmd_present(orig_pmd)) {
+		struct page *page = pmd_page(orig_pmd);
 
-			folio = page_folio(page);
-			folio_remove_rmap_pmd(folio, page, vma);
-			WARN_ON_ONCE(folio_mapcount(folio) < 0);
-			VM_BUG_ON_PAGE(!PageHead(page), page);
-		} else if (pmd_is_valid_softleaf(orig_pmd)) {
-			const softleaf_t entry = softleaf_from_pmd(orig_pmd);
+		folio = page_folio(page);
+		folio_remove_rmap_pmd(folio, page, vma);
+		WARN_ON_ONCE(folio_mapcount(folio) < 0);
+		VM_BUG_ON_PAGE(!PageHead(page), page);
+	} else if (pmd_is_valid_softleaf(orig_pmd)) {
+		const softleaf_t entry = softleaf_from_pmd(orig_pmd);
 
-			folio = softleaf_to_folio(entry);
-			flush_needed = 0;
+		folio = softleaf_to_folio(entry);
+		flush_needed = 0;
 
-			if (!thp_migration_supported())
-				WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
-		}
+		if (!thp_migration_supported())
+			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+	}
 
-		if (folio_test_anon(folio)) {
+	if (folio_test_anon(folio)) {
+		zap_deposited_table(tlb->mm, pmd);
+		add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
+	} else {
+		if (arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
-			add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
-		} else {
-			if (arch_needs_pgtable_deposit())
-				zap_deposited_table(tlb->mm, pmd);
-			add_mm_counter(tlb->mm, mm_counter_file(folio),
-				       -HPAGE_PMD_NR);
-
-			/*
-			 * Use flush_needed to indicate whether the PMD entry
-			 * is present, instead of checking pmd_present() again.
-			 */
-			if (flush_needed && pmd_young(orig_pmd) &&
-			    likely(vma_has_recency(vma)))
-				folio_mark_accessed(folio);
-		}
+		add_mm_counter(tlb->mm, mm_counter_file(folio),
+			       -HPAGE_PMD_NR);
 
-		if (folio_is_device_private(folio)) {
-			folio_remove_rmap_pmd(folio, &folio->page, vma);
-			WARN_ON_ONCE(folio_mapcount(folio) < 0);
-			folio_put(folio);
-		}
+		/*
+		 * Use flush_needed to indicate whether the PMD entry
+		 * is present, instead of checking pmd_present() again.
+		 */
+		if (flush_needed && pmd_young(orig_pmd) &&
+		    likely(vma_has_recency(vma)))
+			folio_mark_accessed(folio);
+	}
 
-		spin_unlock(ptl);
-		if (flush_needed)
-			tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);
+	if (folio_is_device_private(folio)) {
+		folio_remove_rmap_pmd(folio, &folio->page, vma);
+		WARN_ON_ONCE(folio_mapcount(folio) < 0);
+		folio_put(folio);
 	}
+
+	spin_unlock(ptl);
+	if (flush_needed)
+		tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);
+
 	return 1;
 }
 
Signature · 1 line
2.53.0
author reviewer
#11 lore raw

PATCH 3/8 mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc

In reply to Lorenzo Stoakes (Oracle)

There's no need to use the ancient approach of returning an integer here, just return a boolean.

Also update flush_needed to be a boolean, similarly.

Also add a kdoc comment describing the function.

---
 include/linux/huge_mm.h |  4 ++--
 mm/huge_memory.c        | 23 ++++++++++++++++-------
 2 files changed, 18 insertions(+), 9 deletions(-)
include/linux/huge_mm.h +2 -2
index 61fda1672b29..2949e5acff35 100644
@@ -27,8 +27,8 @@ static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
 vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf);
 bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			   pmd_t *pmd, unsigned long addr, unsigned long next);
-int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd,
-		 unsigned long addr);
+bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd,
+		  unsigned long addr);
 int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pud,
 		 unsigned long addr);
 bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
mm/huge_memory.c +16 -7
index 4ebe1f19341e..bba1ba1f6b67 100644
@@ -2418,11 +2418,20 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
 	mm_dec_nr_ptes(mm);
 }
 
-int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
+/**
+ * zap_huge_pmd - Zap a huge THP which is of PMD size.
+ * @tlb: The MMU gather TLB state associated with the operation.
+ * @vma: The VMA containing the range to zap.
+ * @pmd: A pointer to the leaf PMD entry.
+ * @addr: The virtual address for the range to zap.
+ *
+ * Returns: %true on success, %false otherwise.
+ */
+bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		 pmd_t *pmd, unsigned long addr)
 {
 	struct folio *folio = NULL;
-	int flush_needed = 1;
+	bool flush_needed = true;
 	spinlock_t *ptl;
 	pmd_t orig_pmd;
 
@@ -2430,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 
 	ptl = __pmd_trans_huge_lock(pmd, vma);
 	if (!ptl)
-		return 0;
+		return false;
 	/*
 	 * For architectures like ppc64 we look at deposited pgtable
 	 * when calling pmdp_huge_get_and_clear. So do the
@@ -2445,13 +2454,13 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		if (arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
 		spin_unlock(ptl);
-		return 1;
+		return true;
 	}
 	if (is_huge_zero_pmd(orig_pmd)) {
 		if (!vma_is_dax(vma) || arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
 		spin_unlock(ptl);
-		return 1;
+		return true;
 	}
 
 	if (pmd_present(orig_pmd)) {
@@ -2465,7 +2474,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		const softleaf_t entry = softleaf_from_pmd(orig_pmd);
 
 		folio = softleaf_to_folio(entry);
-		flush_needed = 0;
+		flush_needed = false;
 
 		if (!thp_migration_supported())
 			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
@@ -2499,7 +2508,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	if (flush_needed)
 		tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);
 
-	return 1;
+	return true;
 }
 
 #ifndef pmd_move_must_withdraw
Signature · 1 line
2.53.0
author reviewer
#14 lore raw

PATCH 4/8 mm/huge_memory: handle buggy PMD entry in zap_huge_pmd()

In reply to Lorenzo Stoakes (Oracle)

A recent bug I analysed [0] managed to, through a bug in the userfaultfd implementation, reach an invalid point in the zap_huge_pmd() code where the PMD was none of:

- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entry

The code at this point calls folio_test_anon() on a known-NULL folio. Having logic like this explicitly NULL dereference in the code is hard to understand, and makes debugging potentially more difficult.

Add an else branch to handle this case and WARN() and exit indicating failure.

[0]:https://lore.kernel.org/all/[email protected]/

---
 mm/huge_memory.c | 4 ++++
 1 file changed, 4 insertions(+)
mm/huge_memory.c +4 -0
index bba1ba1f6b67..8e6b7ba11448 100644
@@ -2478,6 +2478,10 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 
 		if (!thp_migration_supported())
 			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+	} else {
+		WARN_ON_ONCE(true);
+		spin_unlock(ptl);
+		return false;
 	}
 
 	if (folio_test_anon(folio)) {
Signature · 1 line
2.53.0
author reviewer
#17 lore raw

PATCH 5/8 mm/huge_memory: add a common exit path to zap_huge_pmd()

In reply to Lorenzo Stoakes (Oracle)

Other than when we acquire the PTL, we always need to unlock the PTL, and optionally need to flush on exit.

The code is currently very duplicated in this respect, so default flush_needed to false, set it true in the case in which it's required, then share the same logic for all exit paths.

This also makes flush_needed make more sense as a function-scope value (we don't need to flush for the PFN map/mixed map, zero huge, error cases for instance).

---
 mm/huge_memory.c | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
mm/huge_memory.c +9 -10
index 8e6b7ba11448..724e1de74367 100644
@@ -2431,7 +2431,8 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		 pmd_t *pmd, unsigned long addr)
 {
 	struct folio *folio = NULL;
-	bool flush_needed = true;
+	bool flush_needed = false;
+	bool ret = true;
 	spinlock_t *ptl;
 	pmd_t orig_pmd;
 
@@ -2453,19 +2454,18 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	if (vma_is_special_huge(vma)) {
 		if (arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
-		spin_unlock(ptl);
-		return true;
+		goto out;
 	}
 	if (is_huge_zero_pmd(orig_pmd)) {
 		if (!vma_is_dax(vma) || arch_needs_pgtable_deposit())
 			zap_deposited_table(tlb->mm, pmd);
-		spin_unlock(ptl);
-		return true;
+		goto out;
 	}
 
 	if (pmd_present(orig_pmd)) {
 		struct page *page = pmd_page(orig_pmd);
 
+		flush_needed = true;
 		folio = page_folio(page);
 		folio_remove_rmap_pmd(folio, page, vma);
 		WARN_ON_ONCE(folio_mapcount(folio) < 0);
@@ -2474,14 +2474,13 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		const softleaf_t entry = softleaf_from_pmd(orig_pmd);
 
 		folio = softleaf_to_folio(entry);
-		flush_needed = false;
 
 		if (!thp_migration_supported())
 			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
 	} else {
 		WARN_ON_ONCE(true);
-		spin_unlock(ptl);
-		return false;
+		ret = false;
+		goto out;
 	}
 
 	if (folio_test_anon(folio)) {
@@ -2508,11 +2507,11 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		folio_put(folio);
 	}
 
+out:
 	spin_unlock(ptl);
 	if (flush_needed)
 		tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);
-
-	return true;
+	return ret;
 }
 
 #ifndef pmd_move_must_withdraw
Signature · 1 line
2.53.0
author reviewer
#18 lore raw

PATCH 6/8 mm/huge_memory: remove unnecessary VM_BUG_ON_PAGE()

In reply to Lorenzo Stoakes (Oracle)

This has been around since the beginnings of the THP implementation. I think we can safely assume that, if we have a THP folio, it will have a head page.

---
 mm/huge_memory.c | 1 -
 1 file changed, 1 deletion(-)
mm/huge_memory.c +0 -1
index 724e1de74367..015f6d679d26 100644
@@ -2469,7 +2469,6 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		folio = page_folio(page);
 		folio_remove_rmap_pmd(folio, page, vma);
 		WARN_ON_ONCE(folio_mapcount(folio) < 0);
-		VM_BUG_ON_PAGE(!PageHead(page), page);
 	} else if (pmd_is_valid_softleaf(orig_pmd)) {
 		const softleaf_t entry = softleaf_from_pmd(orig_pmd);
 
Signature · 1 line
2.53.0
author reviewer
#20 lore raw

PATCH 7/8 mm/huge_memory: deduplicate zap deposited table call

In reply to Lorenzo Stoakes (Oracle)

Rather than having separate logic for each case determining whether to zap the deposited table, simply track this via a boolean.

We check separately if the architecture requires it.

Also use pmd_folio() direct in the present case.

---
 mm/huge_memory.c | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)
mm/huge_memory.c +9 -12
index 015f6d679d26..bcc74b0172fa 100644
@@ -2432,6 +2432,7 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 {
 	struct folio *folio = NULL;
 	bool flush_needed = false;
+	bool needs_deposit = false;
 	bool ret = true;
 	spinlock_t *ptl;
 	pmd_t orig_pmd;
@@ -2451,23 +2452,18 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 						tlb->fullmm);
 	arch_check_zapped_pmd(vma, orig_pmd);
 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
-	if (vma_is_special_huge(vma)) {
-		if (arch_needs_pgtable_deposit())
-			zap_deposited_table(tlb->mm, pmd);
+	if (vma_is_special_huge(vma))
 		goto out;
-	}
 	if (is_huge_zero_pmd(orig_pmd)) {
-		if (!vma_is_dax(vma) || arch_needs_pgtable_deposit())
-			zap_deposited_table(tlb->mm, pmd);
+		needs_deposit = !vma_is_dax(vma);
 		goto out;
 	}
 
 	if (pmd_present(orig_pmd)) {
-		struct page *page = pmd_page(orig_pmd);
+		folio = pmd_folio(orig_pmd);
 
 		flush_needed = true;
-		folio = page_folio(page);
-		folio_remove_rmap_pmd(folio, page, vma);
+		folio_remove_rmap_pmd(folio, &folio->page, vma);
 		WARN_ON_ONCE(folio_mapcount(folio) < 0);
 	} else if (pmd_is_valid_softleaf(orig_pmd)) {
 		const softleaf_t entry = softleaf_from_pmd(orig_pmd);
@@ -2483,11 +2479,9 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	}
 
 	if (folio_test_anon(folio)) {
-		zap_deposited_table(tlb->mm, pmd);
+		needs_deposit = true;
 		add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
 	} else {
-		if (arch_needs_pgtable_deposit())
-			zap_deposited_table(tlb->mm, pmd);
 		add_mm_counter(tlb->mm, mm_counter_file(folio),
 			       -HPAGE_PMD_NR);
 
@@ -2507,6 +2501,9 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	}
 
 out:
+	if (arch_needs_pgtable_deposit() || needs_deposit)
+		zap_deposited_table(tlb->mm, pmd);
+
 	spin_unlock(ptl);
 	if (flush_needed)
 		tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);

--
2.53.0

author reviewer
#21 lore raw

PATCH 8/8 mm/huge_memory: deduplicate zap_huge_pmd() further by tracking state

In reply to Lorenzo Stoakes (Oracle)

The flush_needed boolean is really tracking whether a PMD entry is present, so use it that way directly and rename it to is_present.

Deduplicate the folio_remove_rmap_pmd() and folio map count warning between present and device private by tracking where we need to remove the rmap.

We can also remove the comment about using flush_needed to track whether a PMD entry is present as it's now irrelevant.

---
 mm/huge_memory.c | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)
mm/huge_memory.c +13 -15
index bcc74b0172fa..f6caa6d35659 100644
@@ -2430,9 +2430,10 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
 bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		 pmd_t *pmd, unsigned long addr)
 {
+	bool needs_remove_rmap = false;
 	struct folio *folio = NULL;
-	bool flush_needed = false;
 	bool needs_deposit = false;
+	bool is_present = false;
 	bool ret = true;
 	spinlock_t *ptl;
 	pmd_t orig_pmd;
@@ -2450,6 +2451,7 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	 */
 	orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,
 						tlb->fullmm);
+
 	arch_check_zapped_pmd(vma, orig_pmd);
 	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
 	if (vma_is_special_huge(vma))
@@ -2459,17 +2461,15 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		goto out;
 	}
 
-	if (pmd_present(orig_pmd)) {
+	is_present = pmd_present(orig_pmd);
+	if (is_present) {
 		folio = pmd_folio(orig_pmd);
-
-		flush_needed = true;
-		folio_remove_rmap_pmd(folio, &folio->page, vma);
-		WARN_ON_ONCE(folio_mapcount(folio) < 0);
+		needs_remove_rmap = true;
 	} else if (pmd_is_valid_softleaf(orig_pmd)) {
 		const softleaf_t entry = softleaf_from_pmd(orig_pmd);
 
 		folio = softleaf_to_folio(entry);
-
+		needs_remove_rmap = folio_is_device_private(folio);
 		if (!thp_migration_supported())
 			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
 	} else {
@@ -2485,27 +2485,25 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		add_mm_counter(tlb->mm, mm_counter_file(folio),
 			       -HPAGE_PMD_NR);
 
-		/*
-		 * Use flush_needed to indicate whether the PMD entry
-		 * is present, instead of checking pmd_present() again.
-		 */
-		if (flush_needed && pmd_young(orig_pmd) &&
+		if (is_present && pmd_young(orig_pmd) &&
 		    likely(vma_has_recency(vma)))
 			folio_mark_accessed(folio);
 	}
 
-	if (folio_is_device_private(folio)) {
+	if (needs_remove_rmap) {
 		folio_remove_rmap_pmd(folio, &folio->page, vma);
 		WARN_ON_ONCE(folio_mapcount(folio) < 0);
-		folio_put(folio);
 	}
 
 out:
 	if (arch_needs_pgtable_deposit() || needs_deposit)
 		zap_deposited_table(tlb->mm, pmd);
 
+	if (needs_remove_rmap && !is_present)
+		folio_put(folio);
+
 	spin_unlock(ptl);
-	if (flush_needed)
+	if (is_present)
 		tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE);
 	return ret;
 }
Signature · 1 line
2.53.0
#2 lore raw

Re: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()

In reply to Lorenzo Stoakes (Oracle)

mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109

Quoted 12 lines

+++ b/mm/huge_memory.c

mm/huge_memory.c +7 -0
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
 	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
 }
 
+/* If returns true, we are unable to access the VMA's folios. */
+static bool vma_is_special_huge(struct vm_area_struct *vma)
+{
+	if (vma_is_dax(vma))
+		return false;
+	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}

I was hoping that we could make this whole code look more like PTE code by using vm_normal_page_pmd() ... :)

Signature · 3 lines
Cheers,

David
#6 lore raw

Re: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()

In reply to Lorenzo Stoakes (Oracle)
Quoted 27 lines · Lorenzo Stoakes (Oracle)

This function is confused - it overloads the term 'special' yet again, checks for DAX but in many cases the code explicitly excludes DAX before invoking the predicate.

It also unnecessarily checks for vma->vm_file - this has to be present for a driver to have set VMA_MIXEDMAP_BIT or VMA_PFNMAP_BIT.

In fact, a far simpler form of this is to reverse the DAX predicate and return false if DAX is set.

This makes sense from the point of view of 'special' as in vm_normal_page(), as DAX actually does potentially have retrievable folios.

Also there's no need to have this in mm.h so move it to huge_memory.c.

---
  include/linux/huge_mm.h |  4 ++--
  include/linux/mm.h      | 16 ----------------
  mm/huge_memory.c        | 30 +++++++++++++++++++++++-------
  3 files changed, 25 insertions(+), 25 deletions(-)
include/linux/huge_mm.h +0 -0
index bd7f0e1d8094..61fda1672b29 100644

include/linux/huge_mm.h extern struct kobj_attribute thpsize_shmem_enabled_attr; · line 86

Quoted 5 lines
include/linux/huge_mm.h +1 -1
@@ -83,7 +83,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;
   * file is never split and the MAX_PAGECACHE_ORDER limit does not apply to
   * it.  Same to PFNMAPs where there's neither page* nor pagecache.
   */
-#define THP_ORDERS_ALL_SPECIAL		\
+#define THP_ORDERS_ALL_SPECIAL_DAX	\

As mentioned in the comments, the pfnmap case is also include in the 'special' case, right?

Quoted 115 lines
include/linux/huge_mm.h +1 -1
  	(BIT(PMD_ORDER) | BIT(PUD_ORDER))
  #define THP_ORDERS_ALL_FILE_DEFAULT	\
  	((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))
@@ -92,7 +92,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;
   * Mask of all large folio orders supported for THP.
   */
  #define THP_ORDERS_ALL	\
-	(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT)
+	(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT)
 
  enum tva_type {
  	TVA_SMAPS,		/* Exposing "THPeligible:" in smaps. */
include/linux/mm.h +0 -16
index 6f0a3edb24e1..50d68b092204 100644
@@ -5077,22 +5077,6 @@ long copy_folio_from_user(struct folio *dst_folio,
  			   const void __user *usr_src,
  			   bool allow_pagefault);
 
-/**
- * vma_is_special_huge - Are transhuge page-table entries considered special?
- * @vma: Pointer to the struct vm_area_struct to consider
- *
- * Whether transhuge page-table entries are considered "special" following
- * the definition in vm_normal_page().
- *
- * Return: true if transhuge page-table entries should be considered special,
- * false otherwise.
- */
-static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
-{
-	return vma_is_dax(vma) || (vma->vm_file &&
-				   (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
-}
-
  #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
 
  #if MAX_NUMNODES > 1
mm/huge_memory.c +23 -7
index 3fc02913b63e..f76edfa91e96 100644
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
  	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
  }
 
+/* If returns true, we are unable to access the VMA's folios. */
+static bool vma_is_special_huge(struct vm_area_struct *vma)
+{
+	if (vma_is_dax(vma))
+		return false;
+	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}
+
  unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
  					 vm_flags_t vm_flags,
  					 enum tva_type type,
@@ -113,8 +121,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
  	/* Check the intersection of requested and supported orders. */
  	if (vma_is_anonymous(vma))
  		supported_orders = THP_ORDERS_ALL_ANON;
-	else if (vma_is_special_huge(vma))
-		supported_orders = THP_ORDERS_ALL_SPECIAL;
+	else if (vma_is_dax(vma) || vma_is_special_huge(vma))
+		supported_orders = THP_ORDERS_ALL_SPECIAL_DAX;
  	else
  		supported_orders = THP_ORDERS_ALL_FILE_DEFAULT;
 
@@ -2431,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
  						tlb->fullmm);
  	arch_check_zapped_pmd(vma, orig_pmd);
  	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
-	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
+	if (vma_is_special_huge(vma)) {
  		if (arch_needs_pgtable_deposit())
  			zap_deposited_table(tlb->mm, pmd);
  		spin_unlock(ptl);
@@ -2933,7 +2941,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
  	orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm);
  	arch_check_zapped_pud(vma, orig_pud);
  	tlb_remove_pud_tlb_entry(tlb, pud, addr);
-	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
+	if (vma_is_special_huge(vma)) {
  		spin_unlock(ptl);
  		/* No zero page support yet */
  	} else {
@@ -3084,7 +3092,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
  		 */
  		if (arch_needs_pgtable_deposit())
  			zap_deposited_table(mm, pmd);
-		if (!vma_is_dax(vma) && vma_is_special_huge(vma))
+		if (vma_is_special_huge(vma))
  			return;
  		if (unlikely(pmd_is_migration_entry(old_pmd))) {
  			const softleaf_t old_entry = softleaf_from_pmd(old_pmd);
@@ -4645,8 +4653,16 @@ static void split_huge_pages_all(void)
 
  static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
  {
-	return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||
-		    is_vm_hugetlb_page(vma);
+	if (vma_is_dax(vma))
+		return true;
+	if (vma_is_special_huge(vma))
+		return true;
+	if (vma_test(vma, VMA_IO_BIT))
+		return true;
+	if (is_vm_hugetlb_page(vma))
+		return true;
+
+	return false;
  }
 
  static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
Acked-by on v1 2/8
#9 lore raw

Re: [PATCH 2/8] mm/huge: avoid big else branch in zap_huge_pmd()

In reply to Lorenzo Stoakes (Oracle)
Quoted 8 lines · Lorenzo Stoakes (Oracle)

We don't need to have an extra level of indentation, we can simply exit early in the first two branches.

---
  mm/huge_memory.c | 87 +++++++++++++++++++++++++-----------------------
  1 file changed, 45 insertions(+), 42 deletions(-)

The code here isn't too complex, otherwise we can introduce the function like do_zap_pte_range().

Acked-by on v1 3/8
#12 lore raw

Re: [PATCH 3/8] mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc

In reply to Lorenzo Stoakes (Oracle)
Quoted 5 lines · Lorenzo Stoakes (Oracle)

There's no need to use the ancient approach of returning an integer here, just return a boolean.

Also update flush_needed to be a boolean, similarly.

Quoted 8 lines

Also add a kdoc comment describing the function.

---
  include/linux/huge_mm.h |  4 ++--
  mm/huge_memory.c        | 23 ++++++++++++++++-------
  2 files changed, 18 insertions(+), 9 deletions(-)
reviewer Reviewed-by on v1 3/8
#13 lore raw

Re: [PATCH 3/8] mm/huge_memory: have zap_huge_pmd return a boolean, add kdoc

In reply to Lorenzo Stoakes (Oracle)
Quoted 9 lines · Lorenzo Stoakes (Oracle)

There's no need to use the ancient approach of returning an integer here, just return a boolean.

Also update flush_needed to be a boolean, similarly.

Also add a kdoc comment describing the function.

---

Make sense to me.

reviewer
#15 lore raw

Re: [PATCH 4/8] mm/huge_memory: handle buggy PMD entry in zap_huge_pmd()

In reply to Lorenzo Stoakes (Oracle)
Quoted 28 lines · Lorenzo Stoakes (Oracle)

A recent bug I analysed [0] managed to, through a bug in the userfaultfd implementation, reach an invalid point in the zap_huge_pmd() code where the PMD was none of:

- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entry

The code at this point calls folio_test_anon() on a known-NULL folio. Having logic like this explicitly NULL dereference in the code is hard to understand, and makes debugging potentially more difficult.

Add an else branch to handle this case and WARN() and exit indicating failure.

[0]:https://lore.kernel.org/all/[email protected]/

---
  mm/huge_memory.c | 4 ++++
  1 file changed, 4 insertions(+)
mm/huge_memory.c +0 -0
index bba1ba1f6b67..8e6b7ba11448 100644

mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2483

Quoted 6 lines
mm/huge_memory.c +3 -0
@@ -2478,6 +2478,10 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
 
  		if (!thp_migration_supported())
  			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+	} else {
+		WARN_ON_ONCE(true);
+		spin_unlock(ptl);

The warning looks reasonable to me, but ...

mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2484

Quoted 1 line
mm/huge_memory.c +1 -0
+		return false;

IIUC, if we return false here, the caller zap_pmd_range() will fall back to call zap_pte_range(). Since pmd_trans_huge(pmd) returns true, zap_pte_range() will simply return 'addr', causing an infinite loop in zap_pmd_range(), right?

reviewer Reviewed-by on v1 6/8
#19 lore raw

Re: [PATCH 6/8] mm/huge_memory: remove unnecessary VM_BUG_ON_PAGE()

In reply to Lorenzo Stoakes (Oracle)
Quoted 6 lines · Lorenzo Stoakes (Oracle)

This has been around since the beginnings of the THP implementation. I think we can safely assume that, if we have a THP folio, it will have a head page.

---

Yes, it is under the PTL, so the PMD entry should be stable. I also can't think of any case where it can be not a head page.

author reviewer
#3 lore raw

Re: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()

In reply to David Hildenbrand (Arm)
Quoted 5 lines · David Hildenbrand (Arm)
Quoted 5 lines

+++ b/mm/huge_memory.c

mm/huge_memory.c +0 -0
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
 	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
 }
 
Quoted 10 lines

mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109

Quoted 7 lines
mm/huge_memory.c +7 -0
+/* If returns true, we are unable to access the VMA's folios. */
+static bool vma_is_special_huge(struct vm_area_struct *vma)
+{
+	if (vma_is_dax(vma))
+		return false;
+	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}

I was hoping that we could make this whole code look more like PTE code by using vm_normal_page_pmd() ... :)

Could that be a follow up?

I did want to do something where we get the folio and use that to figure things out, but there's a lot of special cases esp. around DAX that make that tricky.

Quoted 5 lines

--
Cheers,

David

Thanks, Lorenzo

author reviewer
#7 lore raw

Re: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()

In reply to Qi Zheng
Quoted 25 lines · Qi Zheng
Quoted 22 lines · Lorenzo Stoakes (Oracle)

This function is confused - it overloads the term 'special' yet again, checks for DAX but in many cases the code explicitly excludes DAX before invoking the predicate.

It also unnecessarily checks for vma->vm_file - this has to be present for a driver to have set VMA_MIXEDMAP_BIT or VMA_PFNMAP_BIT.

In fact, a far simpler form of this is to reverse the DAX predicate and return false if DAX is set.

This makes sense from the point of view of 'special' as in vm_normal_page(), as DAX actually does potentially have retrievable folios.

Also there's no need to have this in mm.h so move it to huge_memory.c.

---
  include/linux/huge_mm.h |  4 ++--
  include/linux/mm.h      | 16 ----------------
  mm/huge_memory.c        | 30 +++++++++++++++++++++++-------
  3 files changed, 25 insertions(+), 25 deletions(-)
Quoted 13 lines

include/linux/huge_mm.h extern struct kobj_attribute thpsize_shmem_enabled_attr; · line 86

Quoted 10 lines
include/linux/huge_mm.h +1 -1
index bd7f0e1d8094..61fda1672b29 100644
@@ -83,7 +83,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;
   * file is never split and the MAX_PAGECACHE_ORDER limit does not apply to
   * it.  Same to PFNMAPs where there's neither page* nor pagecache.
   */
-#define THP_ORDERS_ALL_SPECIAL		\
+#define THP_ORDERS_ALL_SPECIAL_DAX	\

As mentioned in the comments, the pfnmap case is also include in the 'special' case, right?

Yeah special = pfnmap, mixedmap. so renaming to SPECIAL_DAX to make clear it's either dax or 'special' in the meaning of vm_normal_page().

Quoted 110 lines
Quoted 109 lines
include/linux/huge_mm.h +1 -1
  	(BIT(PMD_ORDER) | BIT(PUD_ORDER))
  #define THP_ORDERS_ALL_FILE_DEFAULT	\
  	((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))
@@ -92,7 +92,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr;
   * Mask of all large folio orders supported for THP.
   */
  #define THP_ORDERS_ALL	\
-	(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT)
+	(THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT)
  enum tva_type {
  	TVA_SMAPS,		/* Exposing "THPeligible:" in smaps. */
include/linux/mm.h +0 -16
index 6f0a3edb24e1..50d68b092204 100644
@@ -5077,22 +5077,6 @@ long copy_folio_from_user(struct folio *dst_folio,
  			   const void __user *usr_src,
  			   bool allow_pagefault);
-/**
- * vma_is_special_huge - Are transhuge page-table entries considered special?
- * @vma: Pointer to the struct vm_area_struct to consider
- *
- * Whether transhuge page-table entries are considered "special" following
- * the definition in vm_normal_page().
- *
- * Return: true if transhuge page-table entries should be considered special,
- * false otherwise.
- */
-static inline bool vma_is_special_huge(const struct vm_area_struct *vma)
-{
-	return vma_is_dax(vma) || (vma->vm_file &&
-				   (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
-}
-
  #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
  #if MAX_NUMNODES > 1
mm/huge_memory.c +23 -7
index 3fc02913b63e..f76edfa91e96 100644
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
  	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
  }
+/* If returns true, we are unable to access the VMA's folios. */
+static bool vma_is_special_huge(struct vm_area_struct *vma)
+{
+	if (vma_is_dax(vma))
+		return false;
+	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}
+
  unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
  					 vm_flags_t vm_flags,
  					 enum tva_type type,
@@ -113,8 +121,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
  	/* Check the intersection of requested and supported orders. */
  	if (vma_is_anonymous(vma))
  		supported_orders = THP_ORDERS_ALL_ANON;
-	else if (vma_is_special_huge(vma))
-		supported_orders = THP_ORDERS_ALL_SPECIAL;
+	else if (vma_is_dax(vma) || vma_is_special_huge(vma))
+		supported_orders = THP_ORDERS_ALL_SPECIAL_DAX;
  	else
  		supported_orders = THP_ORDERS_ALL_FILE_DEFAULT;
@@ -2431,7 +2439,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
  						tlb->fullmm);
  	arch_check_zapped_pmd(vma, orig_pmd);
  	tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
-	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
+	if (vma_is_special_huge(vma)) {
  		if (arch_needs_pgtable_deposit())
  			zap_deposited_table(tlb->mm, pmd);
  		spin_unlock(ptl);
@@ -2933,7 +2941,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma,
  	orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm);
  	arch_check_zapped_pud(vma, orig_pud);
  	tlb_remove_pud_tlb_entry(tlb, pud, addr);
-	if (!vma_is_dax(vma) && vma_is_special_huge(vma)) {
+	if (vma_is_special_huge(vma)) {
  		spin_unlock(ptl);
  		/* No zero page support yet */
  	} else {
@@ -3084,7 +3092,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
  		 */
  		if (arch_needs_pgtable_deposit())
  			zap_deposited_table(mm, pmd);
-		if (!vma_is_dax(vma) && vma_is_special_huge(vma))
+		if (vma_is_special_huge(vma))
  			return;
  		if (unlikely(pmd_is_migration_entry(old_pmd))) {
  			const softleaf_t old_entry = softleaf_from_pmd(old_pmd);
@@ -4645,8 +4653,16 @@ static void split_huge_pages_all(void)
  static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma)
  {
-	return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) ||
-		    is_vm_hugetlb_page(vma);
+	if (vma_is_dax(vma))
+		return true;
+	if (vma_is_special_huge(vma))
+		return true;
+	if (vma_test(vma, VMA_IO_BIT))
+		return true;
+	if (is_vm_hugetlb_page(vma))
+		return true;
+
+	return false;
  }
  static int split_huge_pages_pid(int pid, unsigned long vaddr_start,
author reviewer
#16 lore raw

Re: [PATCH 4/8] mm/huge_memory: handle buggy PMD entry in zap_huge_pmd()

In reply to Baolin Wang
Quoted 39 lines · Baolin Wang
Quoted 28 lines · Lorenzo Stoakes (Oracle)

A recent bug I analysed [0] managed to, through a bug in the userfaultfd implementation, reach an invalid point in the zap_huge_pmd() code where the PMD was none of:

- A non-DAX, PFN or mixed map.
- The huge zero folio
- A present PMD entry
- A softleaf entry

The code at this point calls folio_test_anon() on a known-NULL folio. Having logic like this explicitly NULL dereference in the code is hard to understand, and makes debugging potentially more difficult.

Add an else branch to handle this case and WARN() and exit indicating failure.

[0]:https://lore.kernel.org/all/[email protected]/

---
  mm/huge_memory.c | 4 ++++
  1 file changed, 4 insertions(+)
mm/huge_memory.c +0 -0
index bba1ba1f6b67..8e6b7ba11448 100644

mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2482

Quoted 5 lines
mm/huge_memory.c +3 -0
@@ -2478,6 +2478,10 @@ bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
  		if (!thp_migration_supported())
  			WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!");
+	} else {
+		WARN_ON_ONCE(true);
+		spin_unlock(ptl);

The warning looks reasonable to me, but ...

Quoted 6 lines

mm/huge_memory.c bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · line 2484

Quoted 1 line
mm/huge_memory.c +1 -0
+		return false;

IIUC, if we return false here, the caller zap_pmd_range() will fall back to call zap_pte_range(). Since pmd_trans_huge(pmd) returns true, zap_pte_range() will simply return 'addr', causing an infinite loop in zap_pmd_range(), right?

You mean because:

	start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
	if (!pte)
		return addr;

In any case it looks like it degrades a false to potentially carrying on forever:

		addr = zap_pte_range(tlb, vma, pmd, addr, next, details);
		if (addr != next)
			pmd--;

So yeah, this should be a true, annoyingly.

Will fix thanks!

Cheers, Lorenzo

#4 lore raw

Re: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()

In reply to Lorenzo Stoakes (Oracle)
Quoted 17 lines · Lorenzo Stoakes (Oracle)
Quoted 5 lines · David Hildenbrand (Arm)
Quoted 5 lines

+++ b/mm/huge_memory.c

mm/huge_memory.c +0 -0
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
 	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
 }
 
Quoted 10 lines

mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109

Quoted 7 lines
mm/huge_memory.c +7 -0
+/* If returns true, we are unable to access the VMA's folios. */
+static bool vma_is_special_huge(struct vm_area_struct *vma)
+{
+	if (vma_is_dax(vma))
+		return false;
+	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}

I was hoping that we could make this whole code look more like PTE code by using vm_normal_page_pmd() ... :)

Quoted 1 line

Could that be a follow up?

I guess so, but the code is so awful :(

Signature · 3 lines
Cheers,

David
author reviewer
#5 lore raw

Re: [PATCH 1/8] mm/huge_memory: simplify vma_is_specal_huge()

Quoted 18 lines · David Hildenbrand (Arm)
Quoted 17 lines · Lorenzo Stoakes (Oracle)
Quoted 15 lines · David Hildenbrand (Arm)

mm/huge_memory.c static inline bool file_thp_enabled(struct vm_area_struct *vma) · line 109

Quoted 12 lines

+++ b/mm/huge_memory.c

mm/huge_memory.c +7 -0
@@ -100,6 +100,14 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
 	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
 }
 
+/* If returns true, we are unable to access the VMA's folios. */
+static bool vma_is_special_huge(struct vm_area_struct *vma)
+{
+	if (vma_is_dax(vma))
+		return false;
+	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
+}

I was hoping that we could make this whole code look more like PTE code by using vm_normal_page_pmd() ... :)

Quoted 3 lines
Quoted 1 line

Could that be a follow up?

I guess so, but the code is so awful :(

well I udated the code to use vm_normal_folio_pmd() :) at least should get us closer to common logic.

Cheers, Lorenzo