Skip to contentSkip to conversation

arm64: Use pte manipulation functions for THP

PATCH New Steve Capper 4 participants 14 messages Feb 6, 2014 – Feb 20, 2014 arm64

lore
Share & tools
lore
#0 lore raw

cover arm64: Use pte manipulation functions for THP

This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new PTE_WRITE logic, and provides better parity with the HugeTLB code (which already uses the pte functions).

Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.

Steve Capper (3):
arm64: mm: Remove PMD_BIT_FUNC macro
arm64: mm: Route pmd thp functions through pte equivalents
arm64: mm: Correct definition of pmd_mknotpresent

 arch/arm64/include/asm/pgtable.h | 91 +++++++++++++++++++++++++++-------------
 1 file changed, 62 insertions(+), 29 deletions(-)
Signature · 1 line
1.8.1.4
#1 lore raw

PATCH 1/3 arm64: mm: Remove PMD_BIT_FUNC macro

Expand out the pmd thp manipulation functions. This makes our life easier when using things like tags and cscope.

---
 arch/arm64/include/asm/pgtable.h | 51 ++++++++++++++++++++++++++++++++--------
 1 file changed, 41 insertions(+), 10 deletions(-)
arch/arm64/include/asm/pgtable.h +41 -10
index b524dcd..a3fb1e4 100644
@@ -247,16 +247,47 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 #define pmd_trans_splitting(pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING)
 #endif
 
-#define PMD_BIT_FUNC(fn,op) \
-static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; }
-
-PMD_BIT_FUNC(wrprotect,	|= PMD_SECT_RDONLY);
-PMD_BIT_FUNC(mkold,	&= ~PMD_SECT_AF);
-PMD_BIT_FUNC(mksplitting, |= PMD_SECT_SPLITTING);
-PMD_BIT_FUNC(mkwrite,   &= ~PMD_SECT_RDONLY);
-PMD_BIT_FUNC(mkdirty,   |= PMD_SECT_DIRTY);
-PMD_BIT_FUNC(mkyoung,   |= PMD_SECT_AF);
-PMD_BIT_FUNC(mknotpresent, &= ~PMD_TYPE_MASK);
+static inline pmd_t pmd_wrprotect(pmd_t pmd)
+{
+	pmd_val(pmd) |= PMD_SECT_RDONLY;
+	return pmd;
+}
+
+static inline pmd_t pmd_mkold(pmd_t pmd)
+{
+	pmd_val(pmd) &= ~PMD_SECT_AF;
+	return pmd;
+}
+
+static inline pmd_t pmd_mksplitting(pmd_t pmd)
+{
+	pmd_val(pmd) |= PMD_SECT_SPLITTING;
+	return pmd;
+}
+
+static inline pmd_t pmd_mkwrite(pmd_t pmd)
+{
+	pmd_val(pmd) &= ~PMD_SECT_RDONLY;
+	return pmd;
+}
+
+static inline pmd_t pmd_mkdirty(pmd_t pmd)
+{
+	pmd_val(pmd) |= PMD_SECT_DIRTY;
+	return pmd;
+}
+
+static inline pmd_t pmd_mkyoung(pmd_t pmd)
+{
+	pmd_val(pmd) |= PMD_SECT_AF;
+	return pmd;
+}
+
+static inline pmd_t pmd_mknotpresent(pmd_t pmd)
+{
+	pmd_val(pmd) &= ~PMD_TYPE_MASK;
+	return pmd;
+}
 
 #define pmd_mkhuge(pmd)		(__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
 
Signature · 1 line
1.8.1.4
#2 lore raw

PATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents

In reply to Steve Capper

Rather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.

This allows THP to take advantage of the new PTE_WRITE logic introduced in:
c2c93e5 arm64: mm: Introduce PTE_WRITE

To represent splitting THPs we use the PTE_SPECIAL bit as this is not used for pmds.

---
 arch/arm64/include/asm/pgtable.h | 67 +++++++++++++++++++++-------------------
 1 file changed, 35 insertions(+), 32 deletions(-)
arch/arm64/include/asm/pgtable.h +35 -32
index a3fb1e4..a5d5832 100644
@@ -228,59 +228,65 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
 #define __HAVE_ARCH_PTE_SPECIAL
 
 /*
- * Software PMD bits for THP
- */
-
-#define PMD_SECT_DIRTY		(_AT(pmdval_t, 1) << 55)
-#define PMD_SECT_SPLITTING	(_AT(pmdval_t, 1) << 57)
-
-/*
  * THP definitions.
  */
-#define pmd_young(pmd)		(pmd_val(pmd) & PMD_SECT_AF)
-
 #define __HAVE_ARCH_PMD_WRITE
-#define pmd_write(pmd)		(!(pmd_val(pmd) & PMD_SECT_RDONLY))
+static inline long pmd_write(pmd_t pmd)
+{
+	pte_t pte = __pte(pmd_val(pmd));
+	return pte_write(pte);
+}
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 #define pmd_trans_huge(pmd)	(pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
-#define pmd_trans_splitting(pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING)
+
+static inline long pmd_trans_splitting(pmd_t pmd)
+{
+	pte_t pte = __pte(pmd_val(pmd));
+	return pte_special(pte);
+}
 #endif
 
+static inline long pmd_young(pmd_t pmd)
+{
+	pte_t pte = __pte(pmd_val(pmd));
+	return pte_young(pte);
+}
+
 static inline pmd_t pmd_wrprotect(pmd_t pmd)
 {
-	pmd_val(pmd) |= PMD_SECT_RDONLY;
-	return pmd;
+	pte_t pte = pte_wrprotect(__pte(pmd_val(pmd)));
+	return __pmd(pte_val(pte));
 }
 
 static inline pmd_t pmd_mkold(pmd_t pmd)
 {
-	pmd_val(pmd) &= ~PMD_SECT_AF;
-	return pmd;
+	pte_t pte = pte_mkold(__pte(pmd_val(pmd)));
+	return __pmd(pte_val(pte));
 }
 
 static inline pmd_t pmd_mksplitting(pmd_t pmd)
 {
-	pmd_val(pmd) |= PMD_SECT_SPLITTING;
-	return pmd;
+	pte_t pte = pte_mkspecial(__pte(pmd_val(pmd)));
+	return __pmd(pte_val(pte));
 }
 
 static inline pmd_t pmd_mkwrite(pmd_t pmd)
 {
-	pmd_val(pmd) &= ~PMD_SECT_RDONLY;
-	return pmd;
+	pte_t pte = pte_mkwrite(__pte(pmd_val(pmd)));
+	return __pmd(pte_val(pte));
 }
 
 static inline pmd_t pmd_mkdirty(pmd_t pmd)
 {
-	pmd_val(pmd) |= PMD_SECT_DIRTY;
-	return pmd;
+	pte_t pte = pte_mkdirty(__pte(pmd_val(pmd)));
+	return __pmd(pte_val(pte));
 }
 
 static inline pmd_t pmd_mkyoung(pmd_t pmd)
 {
-	pmd_val(pmd) |= PMD_SECT_AF;
-	return pmd;
+	pte_t pte = pte_mkyoung(__pte(pmd_val(pmd)));
+	return __pmd(pte_val(pte));
 }
 
 static inline pmd_t pmd_mknotpresent(pmd_t pmd)
@@ -297,15 +303,6 @@ static inline pmd_t pmd_mknotpresent(pmd_t pmd)
 
 #define pmd_page(pmd)           pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
 
-static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
-{
-	const pmdval_t mask = PMD_SECT_USER | PMD_SECT_PXN | PMD_SECT_UXN |
-			      PMD_SECT_RDONLY | PMD_SECT_PROT_NONE |
-			      PMD_SECT_VALID;
-	pmd_val(pmd) = (pmd_val(pmd) & ~mask) | (pgprot_val(newprot) & mask);
-	return pmd;
-}
-
 #define set_pmd_at(mm, addr, pmdp, pmd)	set_pmd(pmdp, pmd)
 
 static inline int has_transparent_hugepage(void)
@@ -414,6 +411,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
 	return pte;
 }
 
+static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+	pte_t pte = pte_modify(__pte(pmd_val(pmd)), newprot);
+	return __pmd(pte_val(pte));
+}
+
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
 
Signature · 1 line
1.8.1.4
maintainer
#3 lore raw

PATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents

Quoted 3 lines · Steve Capper

Rather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.

That's one thing I don't like about hugetlb, casting pmds to ptes. Do we actually save much in terms of code clean-up by doing this?

Signature · 1 line
Catalin
#4 lore raw

PATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents

Quoted 7 lines · Catalin Marinas
Quoted 3 lines · Steve Capper

Rather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.

That's one thing I don't like about hugetlb, casting pmds to ptes. Do we actually save much in terms of code clean-up by doing this?

Hi Catalin,

At the moment we have ptes and hugetlb ptes behaving in the same way so they both make use of PTE_WRITE. I thought it would be more logical to tie the THP code to the pte manipulation functions too rather than duplicate the PTE_WRITE logic in a separate PMD_WRITE. This makes future pte changes apply to THP too automatically (along with hugetlb).

Cheers,

Signature · 1 line
Steve
maintainer
#5 lore raw

PATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents

Quoted 6 lines · Steve Capper
Quoted 5 lines · Catalin Marinas
Quoted 3 lines · Steve Capper

Rather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.

Quoted 8 lines
Quoted 2 lines

That's one thing I don't like about hugetlb, casting pmds to ptes. Do we actually save much in terms of code clean-up by doing this?

At the moment we have ptes and hugetlb ptes behaving in the same way so they both make use of PTE_WRITE. I thought it would be more logical to tie the THP code to the pte manipulation functions too rather than duplicate the PTE_WRITE logic in a separate PMD_WRITE. This makes future pte changes apply to THP too automatically (along with hugetlb).

OK, let's go with this, it's difficult to change hugetlb the other way.

BTW, powerpc and tile heavily simplify this using macros and pte_pmd/pmd_pte macros.

Signature · 1 line
Catalin
#6 lore raw

PATCH 3/3 arm64: mm: Correct definition of pmd_mknotpresent

In reply to Steve Capper

pmd_mknotpresent currently creates a faulting pmd by clearing the valid bit. Unfortunately pmd_present(.) will report such pmds as being present!

This patch re-defines pmd_mknotpresent such that __pmd(0) is returned instead.

---
 arch/arm64/include/asm/pgtable.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
arch/arm64/include/asm/pgtable.h +1 -2
index a5d5832..5d266e4 100644
@@ -291,8 +291,7 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd)
 
 static inline pmd_t pmd_mknotpresent(pmd_t pmd)
 {
-	pmd_val(pmd) &= ~PMD_TYPE_MASK;
-	return pmd;
+	return __pmd(0);
 }
 
 #define pmd_mkhuge(pmd)		(__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
Signature · 1 line
1.8.1.4
#7 lore raw

cover arm64: Use pte manipulation functions for THP

In reply to Steve Capper
Quoted 7 lines · Steve Capper

This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new PTE_WRITE logic, and provides better parity with the HugeTLB code (which already uses the pte functions).

Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.

Hi,
Does this series look reasonable?

Thanks,

Signature · 1 line
Steve
Quoted 12 lines

Steve Capper (3):
arm64: mm: Remove PMD_BIT_FUNC macro
arm64: mm: Route pmd thp functions through pte equivalents
arm64: mm: Correct definition of pmd_mknotpresent

 arch/arm64/include/asm/pgtable.h | 91 +++++++++++++++++++++++++++-------------
 1 file changed, 62 insertions(+), 29 deletions(-)
Signature · 1 line
1.8.1.4
#8 lore raw

cover arm64: Use pte manipulation functions for THP

Quoted 11 lines · Steve Capper
Quoted 7 lines · Steve Capper

This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new PTE_WRITE logic, and provides better parity with the HugeTLB code (which already uses the pte functions).

Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.

Hi,
Does this series look reasonable?

I was waiting for the corresponding arch/arm/ changes.

Will

maintainer
#9 lore raw

cover arm64: Use pte manipulation functions for THP

Quoted 7 lines · Will Deacon
Quoted 6 lines · Steve Capper
Quoted 5 lines · Steve Capper

This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new PTE_WRITE logic, and provides better parity with the HugeTLB code (which already uses the pte functions).

Quoted 6 lines
Quoted 4 lines
Quoted 2 lines

Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.

Does this series look reasonable?

I was waiting for the corresponding arch/arm/ changes.

The arch/arm/ code doesn't have the PTE_WRITE changes. It's a bit more work here because of the classic MMU (but I hope Steve will get there ;).

Signature · 1 line
Catalin
#10 lore raw

cover arm64: Use pte manipulation functions for THP

Quoted 11 lines · Catalin Marinas
Quoted 10 lines · Will Deacon
Quoted 9 lines · Steve Capper
Quoted 7 lines · Steve Capper

This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new PTE_WRITE logic, and provides better parity with the HugeTLB code (which already uses the pte functions).

Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.

Quoted 6 lines
Quoted 3 lines
Quoted 1 line

Does this series look reasonable?

I was waiting for the corresponding arch/arm/ changes.

The arch/arm/ code doesn't have the PTE_WRITE changes. It's a bit more work here because of the classic MMU (but I hope Steve will get there ;).

Indeed, but I'm wary of divergence in the mm code between the two architectures and I'd much rather we try to keep them in sync, rather than improve/bug fix in one before porting the whole lot over.

Will

#11 lore raw

cover arm64: Use pte manipulation functions for THP

Quoted 14 lines · Will Deacon
Quoted 13 lines · Catalin Marinas
Quoted 12 lines · Will Deacon
Quoted 10 lines · Steve Capper
Quoted 7 lines · Steve Capper

This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new PTE_WRITE logic, and provides better parity with the HugeTLB code (which already uses the pte functions).

Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.

Does this series look reasonable?

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

I was waiting for the corresponding arch/arm/ changes.

The arch/arm/ code doesn't have the PTE_WRITE changes. It's a bit more work here because of the classic MMU (but I hope Steve will get there ;).

Indeed, but I'm wary of divergence in the mm code between the two architectures and I'd much rather we try to keep them in sync, rather than improve/bug fix in one before porting the whole lot over.

I do think we need the same behaviour for 3-level code on ARM and for ARM64.

The PTE_WRITE changes for ARM64 allow one to distinguish between writable clean ptes and read only ptes which fixes a very subtle problem with huge pages (and maybe some other problems). This problem is still present in the ARM 3-level code.

The ARM 2-level code stores a separate linux/hardware pte and it is possible to distinguish between writable clean ptes and read only ptes.

For ARM, I would like to split out some of the pte logic and page permissions from pgtable.h into pgtable-[23]level.h. Then introduce PTE_WRITE for the 3-level code on ARM.

Russell,
I have noticed the following for ARM:
36bb94b ARM: pgtable: provide RDONLY page table bit rather than WRITE bit
Are you happy for me to re-introduce PTE_WRITE for the 3-level code only, and keep L_PTE_RDONLY for the 2-level code?

Thanks,

Signature · 1 line
Steve
#12 lore raw

cover arm64: Use pte manipulation functions for THP

Quoted 4 lines · Steve Capper

I have noticed the following for ARM:
36bb94b ARM: pgtable: provide RDONLY page table bit rather than WRITE bit
Are you happy for me to re-introduce PTE_WRITE for the 3-level code only, and keep L_PTE_RDONLY for the 2-level code?

Sigh, yet another cleanup reverted... If you must.

Signature · 3 lines
FTTC broadband for 0.8mile line: 5.8Mbps down 500kbps up.  Estimation
in database were 13.1 to 19Mbit for a good line, about 7.5+ for a bad.
Estimate before purchase was "up to 13.2Mbit".
maintainer
#13 lore raw

cover arm64: Use pte manipulation functions for THP

In reply to Steve Capper
Quoted 4 lines · Steve Capper

I have noticed the following for ARM:
36bb94b ARM: pgtable: provide RDONLY page table bit rather than WRITE bit
Are you happy for me to re-introduce PTE_WRITE for the 3-level code only, and keep L_PTE_RDONLY for the 2-level code?

But don't we create more confusion by differentiating between 2-level and 3-level? What about L_PTE_WRITE in all cases as pure software bit and PTE_RDONLY (without L_) defined just for LPAE.

Signature · 1 line
Catalin