Skip to contentSkip to conversation

powerpc/mm/hugetlb: Update huge_ptep_set_access_flags to call __ptep_set_access_flags directly

PATCH In review Aneesh Kumar K.V 3 participants 6 messages May 29, 2018 – Jun 4, 2018 PowerPC

lore
Share & tools
lore

Aneesh Kumar K.V

author
#0 lore raw

PATCH v2 1/4 powerpc/mm/hugetlb: Update huge_ptep_set_access_flags to call __ptep_set_access_flags directly

In a later patch, we want to update __ptep_set_access_flags take page size arg. This makes ptep_set_access_flags only work with mmu_virtual_psize. To simplify the code make huge_ptep_set_access_flags directly call __ptep_set_access_flags so that we can compute the hugetlb page size in hugetlb function.

Now that ptep_set_access_flags won't be called for hugetlb remove the is_vm_hugetlb_page() check and add the assert of pte lock unconditionally.

Signed-off-by Aneesh Kumar K.V <[email protected]>
---
 arch/powerpc/include/asm/hugetlb.h | 19 +++--------------
 arch/powerpc/mm/pgtable.c          | 33 ++++++++++++++++++++++++++++--
 2 files changed, 34 insertions(+), 18 deletions(-)
arch/powerpc/include/asm/hugetlb.h +3 -16
index 78540c074d70..b4404a6da74f 100644
@@ -166,22 +166,9 @@ static inline pte_t huge_pte_wrprotect(pte_t pte)
 	return pte_wrprotect(pte);
 }
 
-static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
-					     unsigned long addr, pte_t *ptep,
-					     pte_t pte, int dirty)
-{
-#ifdef HUGETLB_NEED_PRELOAD
-	/*
-	 * The "return 1" forces a call of update_mmu_cache, which will write a
-	 * TLB entry.  Without this, platforms that don't do a write of the TLB
-	 * entry in the TLB miss handler asm will fault ad infinitum.
-	 */
-	ptep_set_access_flags(vma, addr, ptep, pte, dirty);
-	return 1;
-#else
-	return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
-#endif
-}
+extern int huge_ptep_set_access_flags(struct vm_area_struct *vma,
+				      unsigned long addr, pte_t *ptep,
+				      pte_t pte, int dirty);
 
 static inline pte_t huge_ptep_get(pte_t *ptep)
 {
arch/powerpc/mm/pgtable.c +31 -2
index 9f361ae571e9..e70af9939379 100644
@@ -221,14 +221,43 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 	entry = set_access_flags_filter(entry, vma, dirty);
 	changed = !pte_same(*(ptep), entry);
 	if (changed) {
-		if (!is_vm_hugetlb_page(vma))
-			assert_pte_locked(vma->vm_mm, address);
+		assert_pte_locked(vma->vm_mm, address);
 		__ptep_set_access_flags(vma->vm_mm, ptep, entry, address);
 		flush_tlb_page(vma, address);
 	}
 	return changed;
 }
 
+#ifdef CONFIG_HUGETLB_PAGE
+extern int huge_ptep_set_access_flags(struct vm_area_struct *vma,
+				      unsigned long addr, pte_t *ptep,
+				      pte_t pte, int dirty)
+{
+#ifdef HUGETLB_NEED_PRELOAD
+	/*
+	 * The "return 1" forces a call of update_mmu_cache, which will write a
+	 * TLB entry.  Without this, platforms that don't do a write of the TLB
+	 * entry in the TLB miss handler asm will fault ad infinitum.
+	 */
+	ptep_set_access_flags(vma, addr, ptep, pte, dirty);
+	return 1;
+#else
+	int changed;
+
+	pte = set_access_flags_filter(pte, vma, dirty);
+	changed = !pte_same(*(ptep), pte);
+	if (changed) {
+#ifdef CONFIG_DEBUG_VM
+		assert_spin_locked(&vma->vm_mm->page_table_lock);
+#endif
+		__ptep_set_access_flags(vma->vm_mm, ptep, pte, addr);
+		flush_hugetlb_page(vma, addr);
+	}
+	return changed;
+#endif
+}
+#endif /* CONFIG_HUGETLB_PAGE */
+
 #ifdef CONFIG_DEBUG_VM
 void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
 {
Signature · 1 line
2.17.0
author
#1 lore raw

PATCH v2 2/4 powerpc/mm/radix: Move function from radix.h to pgtable-radix.c

In later patch we will update them which require them to be moved to pgtable-radix.c. Keeping the function in radix.h results in compile warning as below.

./arch/powerpc/include/asm/book3s/64/radix.h: In function ‘radix__ptep_set_access_flags’:
./arch/powerpc/include/asm/book3s/64/radix.h:196:28: error: dereferencing pointer to incomplete type ‘struct vm_area_struct’

  struct mm_struct *mm = vma->vm_mm;
                            ^~
./arch/powerpc/include/asm/book3s/64/radix.h:204:6: error: implicit declaration of function ‘atomic_read’; did you mean ‘__atomic_load’? [-Werror=implicit-function-declaration]
      atomic_read(&mm->context.copros) > 0) {
      ^~~~~~~~~~~
      __atomic_load
./arch/powerpc/include/asm/book3s/64/radix.h:204:21: error: dereferencing pointer to incomplete type ‘struct mm_struct’
      atomic_read(&mm->context.copros) > 0) {

Instead of fixing header dependencies, we move the function to pgtable-radix.c Also the function is now large to be a static inline . Doing the move in separate patch helps in review.

No functional change in this patch. Only code movement.

Signed-off-by Aneesh Kumar K.V <[email protected]>
---
 arch/powerpc/include/asm/book3s/64/radix.h | 31 +++-------------------
 arch/powerpc/mm/pgtable-radix.c            | 22 +++++++++++++++
 2 files changed, 25 insertions(+), 28 deletions(-)
arch/powerpc/include/asm/book3s/64/radix.h +3 -28
index 705193e7192f..36ed025b4e13 100644
@@ -124,6 +124,9 @@ extern void radix__mark_rodata_ro(void);
 extern void radix__mark_initmem_nx(void);
 #endif
 
+extern void radix__ptep_set_access_flags(struct mm_struct *mm, pte_t *ptep,
+					 pte_t entry, unsigned long address);
+
 static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr,
 					       unsigned long set)
 {

@@ -190,34 +193,6 @@ static inline pte_t radix__ptep_get_and_clear_full(struct mm_struct *mm,
 	return __pte(old_pte);
 }
 
-/*
- * Set the dirty and/or accessed bits atomically in a linux PTE, this
- * function doesn't need to invalidate tlb.
- */
-static inline void radix__ptep_set_access_flags(struct mm_struct *mm,
-						pte_t *ptep, pte_t entry,
-						unsigned long address)
-{
-
-	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
-					      _PAGE_RW | _PAGE_EXEC);
-
-	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
-
-		unsigned long old_pte, new_pte;
-
-		old_pte = __radix_pte_update(ptep, ~0, 0);
-		/*
-		 * new value of pte
-		 */
-		new_pte = old_pte | set;
-		radix__flush_tlb_pte_p9_dd1(old_pte, mm, address);
-		__radix_pte_update(ptep, 0, new_pte);
-	} else
-		__radix_pte_update(ptep, 0, set);
-	asm volatile("ptesync" : : : "memory");
-}
-
 static inline int radix__pte_same(pte_t pte_a, pte_t pte_b)
 {
 	return ((pte_raw(pte_a) ^ pte_raw(pte_b)) == 0);
arch/powerpc/mm/pgtable-radix.c +22 -0
index ce24d72ea679..a6eec41dd347 100644
@@ -1084,3 +1084,25 @@ int radix__has_transparent_hugepage(void)
 	return 0;
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
+void radix__ptep_set_access_flags(struct mm_struct *mm,
+				  pte_t *ptep, pte_t entry,
+				  unsigned long address)
+{
+	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
+					      _PAGE_RW | _PAGE_EXEC);
+
+	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
+		unsigned long old_pte, new_pte;
+
+		old_pte = __radix_pte_update(ptep, ~0, 0);
+		/*
+		 * new value of pte
+		 */
+		new_pte = old_pte | set;
+		radix__flush_tlb_pte_p9_dd1(old_pte, mm, address);
+		__radix_pte_update(ptep, 0, new_pte);
+	} else
+		__radix_pte_update(ptep, 0, set);
+	asm volatile("ptesync" : : : "memory");
+}
Signature · 1 line
2.17.0
author
#2 lore raw

PATCH v2 3/4 powerpc/mm: Change function prototype

In reply to Aneesh Kumar K.V

In later patch, we use the vma and psize to do tlb flush. Do the prototype update in separate patch to make the review easy.

Signed-off-by Aneesh Kumar K.V <[email protected]>
---
 arch/powerpc/include/asm/book3s/32/pgtable.h |  5 +++--
 arch/powerpc/include/asm/book3s/64/pgtable.h |  8 +++++---
 arch/powerpc/include/asm/book3s/64/radix.h   |  5 +++--
 arch/powerpc/include/asm/nohash/32/pgtable.h |  5 +++--
 arch/powerpc/include/asm/nohash/64/pgtable.h |  5 +++--
 arch/powerpc/mm/pgtable-book3s64.c           |  8 ++++++--
 arch/powerpc/mm/pgtable-radix.c              |  6 +++---
 arch/powerpc/mm/pgtable.c                    | 18 +++++++++++++++---
 8 files changed, 41 insertions(+), 19 deletions(-)
arch/powerpc/include/asm/book3s/32/pgtable.h +3 -2
index c615abdce119..39d3a4245694 100644
@@ -235,9 +235,10 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
 }
 
 
-static inline void __ptep_set_access_flags(struct mm_struct *mm,
+static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
 					   pte_t *ptep, pte_t entry,
-					   unsigned long address)
+					   unsigned long address,
+					   int psize)
 {
 	unsigned long set = pte_val(entry) &
 		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
arch/powerpc/include/asm/book3s/64/pgtable.h +5 -3
index c233915abb68..42fe7c2ff2df 100644
@@ -767,12 +767,14 @@ static inline bool check_pte_access(unsigned long access, unsigned long ptev)
  * Generic functions with hash/radix callbacks
  */
 
-static inline void __ptep_set_access_flags(struct mm_struct *mm,
+static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
 					   pte_t *ptep, pte_t entry,
-					   unsigned long address)
+					   unsigned long address,
+					   int psize)
 {
 	if (radix_enabled())
-		return radix__ptep_set_access_flags(mm, ptep, entry, address);
+		return radix__ptep_set_access_flags(vma, ptep, entry,
+						    address, psize);
 	return hash__ptep_set_access_flags(ptep, entry);
 }
arch/powerpc/include/asm/book3s/64/radix.h +3 -2
index 36ed025b4e13..62a73a7a78a4 100644
@@ -124,8 +124,9 @@ extern void radix__mark_rodata_ro(void);
 extern void radix__mark_initmem_nx(void);
 #endif
 
-extern void radix__ptep_set_access_flags(struct mm_struct *mm, pte_t *ptep,
-					 pte_t entry, unsigned long address);
+extern void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
+					 pte_t entry, unsigned long address,
+					 int psize);
 
 static inline unsigned long __radix_pte_update(pte_t *ptep, unsigned long clr,
 					       unsigned long set)
arch/powerpc/include/asm/nohash/32/pgtable.h +3 -2
index 987a658b18e1..c2471bac86b9 100644
@@ -256,9 +256,10 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
 }
 
 
-static inline void __ptep_set_access_flags(struct mm_struct *mm,
+static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
 					   pte_t *ptep, pte_t entry,
-					   unsigned long address)
+					   unsigned long address,
+					   int psize)
 {
 	unsigned long set = pte_val(entry) &
 		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
arch/powerpc/include/asm/nohash/64/pgtable.h +3 -2
index de78eda5f841..180161d714fb 100644
@@ -281,9 +281,10 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
 /* Set the dirty and/or accessed bits atomically in a linux PTE, this
  * function doesn't need to flush the hash entry
  */
-static inline void __ptep_set_access_flags(struct mm_struct *mm,
+static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
 					   pte_t *ptep, pte_t entry,
-					   unsigned long address)
+					   unsigned long address,
+					   int psize)
 {
 	unsigned long bits = pte_val(entry) &
 		(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC);
arch/powerpc/mm/pgtable-book3s64.c +6 -2
index abda2b92f1ba..4a8150481a88 100644
@@ -46,8 +46,12 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 #endif
 	changed = !pmd_same(*(pmdp), entry);
 	if (changed) {
-		__ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp),
-					pmd_pte(entry), address);
+		/*
+		 * We can use MMU_PAGE_2M here, because only radix
+		 * path look at the psize.
+		 */
+		__ptep_set_access_flags(vma, pmdp_ptep(pmdp),
+					pmd_pte(entry), address, MMU_PAGE_2M);
 		flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 	}
 	return changed;
arch/powerpc/mm/pgtable-radix.c +3 -3
index a6eec41dd347..2034cbc9aa56 100644
@@ -1085,10 +1085,10 @@ int radix__has_transparent_hugepage(void)
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
-void radix__ptep_set_access_flags(struct mm_struct *mm,
-				  pte_t *ptep, pte_t entry,
-				  unsigned long address)
+void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
+				  pte_t entry, unsigned long address, int psize)
 {
+	struct mm_struct *mm = vma->vm_mm;
 	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
 					      _PAGE_RW | _PAGE_EXEC);
arch/powerpc/mm/pgtable.c +15 -3
index e70af9939379..20cacd33e5be 100644
@@ -222,7 +222,8 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 	changed = !pte_same(*(ptep), entry);
 	if (changed) {
 		assert_pte_locked(vma->vm_mm, address);
-		__ptep_set_access_flags(vma->vm_mm, ptep, entry, address);
+		__ptep_set_access_flags(vma, ptep, entry,
+					address, mmu_virtual_psize);
 		flush_tlb_page(vma, address);
 	}
 	return changed;

@@ -242,15 +243,26 @@ extern int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 	ptep_set_access_flags(vma, addr, ptep, pte, dirty);
 	return 1;
 #else
-	int changed;
+	int changed, psize;
 
 	pte = set_access_flags_filter(pte, vma, dirty);
 	changed = !pte_same(*(ptep), pte);
 	if (changed) {
+
+#ifdef CONFIG_PPC_BOOK3S_64
+		struct hstate *hstate = hstate_file(vma->vm_file);
+		psize = hstate_get_psize(hstate);
+#else
+		/*
+		 * Not used on non book3s64 platforms. But 8xx
+		 * can possibly use tsize derived from hstate.
+		 */
+		psize = 0;
+#endif
 #ifdef CONFIG_DEBUG_VM
 		assert_spin_locked(&vma->vm_mm->page_table_lock);
 #endif
-		__ptep_set_access_flags(vma->vm_mm, ptep, pte, addr);
+		__ptep_set_access_flags(vma, ptep, pte, addr, psize);
 		flush_hugetlb_page(vma, addr);
 	}
 	return changed;
Signature · 1 line
2.17.0
author
#3 lore raw

PATCH v2 4/4 powerpc/mm/radix: Change pte relax sequence to handle nest MMU hang

In reply to Aneesh Kumar K.V

When relaxing access (read -> read_write update), pte needs to be marked invalid to handle a nest MMU bug. We also need to do a tlb flush after the pte is marked invalid before updating the pte with new access bits.

We also move tlb flush to platform specific __ptep_set_access_flags. This will help us to gerid of unnecessary tlb flush on BOOK3S 64 later. We don't do that in this patch. This also helps in avoiding multiple tlbies with coprocessor attached.

Signed-off-by Aneesh Kumar K.V <[email protected]>
---
 arch/powerpc/include/asm/book3s/32/pgtable.h |  2 ++
 arch/powerpc/include/asm/nohash/32/pgtable.h |  2 ++
 arch/powerpc/include/asm/nohash/64/pgtable.h |  2 ++
 arch/powerpc/include/asm/pgtable.h           |  1 +
 arch/powerpc/mm/pgtable-book3s64.c           |  1 -
 arch/powerpc/mm/pgtable-radix.c              | 14 ++++++++++----
 arch/powerpc/mm/pgtable.c                    |  2 --
 7 files changed, 17 insertions(+), 7 deletions(-)
arch/powerpc/include/asm/book3s/32/pgtable.h +2 -0
index 39d3a4245694..02f5acd7ccc4 100644
@@ -245,6 +245,8 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
 	unsigned long clr = ~pte_val(entry) & _PAGE_RO;
 
 	pte_update(ptep, clr, set);
+
+	flush_tlb_page(vma, address);
 }
 
 #define __HAVE_ARCH_PTE_SAME
arch/powerpc/include/asm/nohash/32/pgtable.h +2 -0
index c2471bac86b9..7c46a98cc7f4 100644
@@ -266,6 +266,8 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
 	unsigned long clr = ~pte_val(entry) & (_PAGE_RO | _PAGE_NA);
 
 	pte_update(ptep, clr, set);
+
+	flush_tlb_page(vma, address);
 }
 
 static inline int pte_young(pte_t pte)
arch/powerpc/include/asm/nohash/64/pgtable.h +2 -0
index 180161d714fb..dd0c7236208f 100644
@@ -304,6 +304,8 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma,
 	unsigned long old = pte_val(*ptep);
 	*ptep = __pte(old | bits);
 #endif
+
+	flush_tlb_page(vma, address);
 }
 
 #define __HAVE_ARCH_PTE_SAME
arch/powerpc/include/asm/pgtable.h +1 -0
index ab7d2d996be4..14c79a7dc855 100644
@@ -8,6 +8,7 @@
 #include <asm/processor.h>		/* For TASK_SIZE */
 #include <asm/mmu.h>
 #include <asm/page.h>
+#include <asm/tlbflush.h>
 
 struct mm_struct;
arch/powerpc/mm/pgtable-book3s64.c +0 -1
index 4a8150481a88..82fed87289de 100644
@@ -52,7 +52,6 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 		 */
 		__ptep_set_access_flags(vma, pmdp_ptep(pmdp),
 					pmd_pte(entry), address, MMU_PAGE_2M);
-		flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 	}
 	return changed;
 }
arch/powerpc/mm/pgtable-radix.c +10 -4
index 2034cbc9aa56..0ddfe591cd24 100644
@@ -1091,8 +1091,12 @@ void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
 	struct mm_struct *mm = vma->vm_mm;
 	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
 					      _PAGE_RW | _PAGE_EXEC);
-
-	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
+	/*
+	 * To avoid NMMU hang while relaxing access, we need mark
+	 * the pte invalid in between.
+	 */
+	if (cpu_has_feature(CPU_FTR_POWER9_DD1) ||
+	    atomic_read(&mm->context.copros) > 0) {
 		unsigned long old_pte, new_pte;
 
 		old_pte = __radix_pte_update(ptep, ~0, 0);

@@ -1100,9 +1104,11 @@ void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
 		 * new value of pte
 		 */
 		new_pte = old_pte | set;
-		radix__flush_tlb_pte_p9_dd1(old_pte, mm, address);
+		radix__flush_tlb_page_psize(mm, address, psize);
 		__radix_pte_update(ptep, 0, new_pte);
-	} else
+	} else {
 		__radix_pte_update(ptep, 0, set);
+		radix__flush_tlb_page_psize(mm, address, psize);
+	}
 	asm volatile("ptesync" : : : "memory");
 }
arch/powerpc/mm/pgtable.c +0 -2
index 20cacd33e5be..5281c2c064af 100644
@@ -224,7 +224,6 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
 		assert_pte_locked(vma->vm_mm, address);
 		__ptep_set_access_flags(vma, ptep, entry,
 					address, mmu_virtual_psize);
-		flush_tlb_page(vma, address);
 	}
 	return changed;
 }

@@ -263,7 +262,6 @@ extern int huge_ptep_set_access_flags(struct vm_area_struct *vma,
 		assert_spin_locked(&vma->vm_mm->page_table_lock);
 #endif
 		__ptep_set_access_flags(vma, ptep, pte, addr, psize);
-		flush_hugetlb_page(vma, addr);
 	}
 	return changed;
 #endif
Signature · 1 line
2.17.0
reviewer
#4 lore raw

Re: [PATCH V2 4/4] powerpc/mm/radix: Change pte relax sequence to handle nest MMU hang

Quoted 8 lines · On Tue, 29 May 2018 19:58:41 +0530 "Aneesh Kumar K.V"…

When relaxing access (read -> read_write update), pte needs to be marked invalid to handle a nest MMU bug. We also need to do a tlb flush after the pte is marked invalid before updating the pte with new access bits.

We also move tlb flush to platform specific __ptep_set_access_flags. This will help us to gerid of unnecessary tlb flush on BOOK3S 64 later. We don't do that in this patch. This also helps in avoiding multiple tlbies with coprocessor attached.

I thought you should move this part of the change back into patch 3 as well.

Thanks,
Nick

#5 lore raw
In reply to Aneesh Kumar K.V
Quoted 11 lines · Aneesh Kumar K.V

In a later patch, we want to update __ptep_set_access_flags take page size arg. This makes ptep_set_access_flags only work with mmu_virtual_psize. To simplify the code make huge_ptep_set_access_flags directly call __ptep_set_access_flags so that we can compute the hugetlb page size in hugetlb function.

Now that ptep_set_access_flags won't be called for hugetlb remove the is_vm_hugetlb_page() check and add the assert of pte lock unconditionally.

Signed-off-by Aneesh Kumar K.V <[email protected]>

Series applied to powerpc next, thanks.

https://git.kernel.org/powerpc/c/f069ff396d657ac7bdb5de866c3ec2

cheers