arm64: Use pte manipulation functions for THP
cover arm64: Use pte manipulation functions for THP
This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new PTE_WRITE logic, and provides better parity with the HugeTLB code (which already uses the pte functions).
Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.
Steve Capper (3):
arm64: mm: Remove PMD_BIT_FUNC macro
arm64: mm: Route pmd thp functions through pte equivalents
arm64: mm: Correct definition of pmd_mknotpresent
arch/arm64/include/asm/pgtable.h | 91 +++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 29 deletions(-)
Signature · 1 line
1.8.1.4
PATCH 1/3 arm64: mm: Remove PMD_BIT_FUNC macro
Expand out the pmd thp manipulation functions. This makes our life easier when using things like tags and cscope.
--- arch/arm64/include/asm/pgtable.h | 51 ++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 10 deletions(-)
arch/arm64/include/asm/pgtable.h +41 -10
@@ -247,16 +247,47 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,#define pmd_trans_splitting (pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING)#endif -#define PMD_BIT_FUNC (fn,op) \ -static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } - -PMD_BIT_FUNC(wrprotect, |= PMD_SECT_RDONLY); -PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF); -PMD_BIT_FUNC(mksplitting, |= PMD_SECT_SPLITTING); -PMD_BIT_FUNC(mkwrite, &= ~PMD_SECT_RDONLY); -PMD_BIT_FUNC(mkdirty, |= PMD_SECT_DIRTY); -PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF); -PMD_BIT_FUNC(mknotpresent, &= ~PMD_TYPE_MASK); +static inline pmd_t pmd_wrprotect (pmd_t pmd) +{ +pmd_val (pmd) |=PMD_SECT_RDONLY ; +return pmd; +} + +static inline pmd_t pmd_mkold (pmd_t pmd) +{ +pmd_val (pmd) &= ~PMD_SECT_AF ; +return pmd; +} + +static inline pmd_t pmd_mksplitting (pmd_t pmd) +{ +pmd_val (pmd) |=PMD_SECT_SPLITTING ; +return pmd; +} + +static inline pmd_t pmd_mkwrite (pmd_t pmd) +{ +pmd_val (pmd) &= ~PMD_SECT_RDONLY ; +return pmd; +} + +static inline pmd_t pmd_mkdirty (pmd_t pmd) +{ +pmd_val (pmd) |=PMD_SECT_DIRTY ; +return pmd; +} + +static inline pmd_t pmd_mkyoung (pmd_t pmd) +{ +pmd_val (pmd) |=PMD_SECT_AF ; +return pmd; +} + +static inline pmd_t pmd_mknotpresent (pmd_t pmd) +{ +pmd_val (pmd) &= ~PMD_TYPE_MASK ; +return pmd; +}#define pmd_mkhuge (pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
Signature · 1 line
1.8.1.4
PATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents
In reply to Steve CapperRather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.
This allows THP to take advantage of the new PTE_WRITE logic introduced in:
c2c93e5 arm64: mm: Introduce PTE_WRITE
To represent splitting THPs we use the PTE_SPECIAL bit as this is not used for pmds.
--- arch/arm64/include/asm/pgtable.h | 67 +++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 32 deletions(-)
arch/arm64/include/asm/pgtable.h +35 -32
@@ -228,59 +228,65 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,#define __HAVE_ARCH_PTE_SPECIAL/* -* Software PMD bits for THP -*/ - -#define PMD_SECT_DIRTY (_AT(pmdval_t, 1) << 55) -#define PMD_SECT_SPLITTING (_AT(pmdval_t, 1) << 57) - -/* * THP definitions. */ -#define pmd_young (pmd) (pmd_val(pmd) & PMD_SECT_AF) -#define __HAVE_ARCH_PMD_WRITE -#define pmd_write (pmd) (!(pmd_val(pmd) & PMD_SECT_RDONLY)) +static inline long pmd_write (pmd_t pmd) +{ +pte_t pte =__pte (pmd_val (pmd)); +return pte_write (pte); +}#ifdef CONFIG_TRANSPARENT_HUGEPAGE #define pmd_trans_huge (pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT)) -#define pmd_trans_splitting (pmd) (pmd_val(pmd) & PMD_SECT_SPLITTING) + +static inline long pmd_trans_splitting (pmd_t pmd) +{ +pte_t pte =__pte (pmd_val (pmd)); +return pte_special (pte); +}#endif +static inline long pmd_young (pmd_t pmd) +{ +pte_t pte =__pte (pmd_val (pmd)); +return pte_young (pte); +} +static inline pmd_t pmd_wrprotect (pmd_t pmd) { -pmd_val (pmd) |=PMD_SECT_RDONLY ; -return pmd; +pte_t pte =pte_wrprotect (__pte (pmd_val (pmd))); +return __pmd (pte_val (pte)); }static inline pmd_t pmd_mkold (pmd_t pmd) { -pmd_val (pmd) &= ~PMD_SECT_AF ; -return pmd; +pte_t pte =pte_mkold (__pte (pmd_val (pmd))); +return __pmd (pte_val (pte)); }static inline pmd_t pmd_mksplitting (pmd_t pmd) { -pmd_val (pmd) |=PMD_SECT_SPLITTING ; -return pmd; +pte_t pte =pte_mkspecial (__pte (pmd_val (pmd))); +return __pmd (pte_val (pte)); }static inline pmd_t pmd_mkwrite (pmd_t pmd) { -pmd_val (pmd) &= ~PMD_SECT_RDONLY ; -return pmd; +pte_t pte =pte_mkwrite (__pte (pmd_val (pmd))); +return __pmd (pte_val (pte)); }static inline pmd_t pmd_mkdirty (pmd_t pmd) { -pmd_val (pmd) |=PMD_SECT_DIRTY ; -return pmd; +pte_t pte =pte_mkdirty (__pte (pmd_val (pmd))); +return __pmd (pte_val (pte)); }static inline pmd_t pmd_mkyoung (pmd_t pmd) { -pmd_val (pmd) |=PMD_SECT_AF ; -return pmd; +pte_t pte =pte_mkyoung (__pte (pmd_val (pmd))); +return __pmd (pte_val (pte)); }static inline pmd_t pmd_mknotpresent (pmd_t pmd)
@@ -297,15 +303,6 @@ static inline pmd_t pmd_mknotpresent(pmd_t pmd)#define pmd_page (pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) -static inline pmd_t pmd_modify (pmd_t pmd,pgprot_t newprot) -{ -const pmdval_t mask =PMD_SECT_USER |PMD_SECT_PXN |PMD_SECT_UXN | -PMD_SECT_RDONLY |PMD_SECT_PROT_NONE | -PMD_SECT_VALID ; -pmd_val (pmd) = (pmd_val (pmd) & ~mask) | (pgprot_val (newprot) & mask); -return pmd; -} -#define set_pmd_at (mm, addr, pmdp, pmd) set_pmd(pmdp, pmd)static inline int has_transparent_hugepage (void )
@@ -414,6 +411,12 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)return pte; } +static inline pmd_t pmd_modify (pmd_t pmd,pgprot_t newprot) +{ +pte_t pte =pte_modify (__pte (pmd_val (pmd)), newprot); +return __pmd (pte_val (pte)); +} +extern pgd_t swapper_pg_dir[PTRS_PER_PGD ];extern pgd_t idmap_pg_dir[PTRS_PER_PGD ];
Signature · 1 line
1.8.1.4
Catalin Marinas
maintainerPATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents
Quoted 3 lines · Steve Capper
Rather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.
That's one thing I don't like about hugetlb, casting pmds to ptes. Do we actually save much in terms of code clean-up by doing this?
Signature · 1 line
Catalin
PATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents
Quoted 7 lines · Catalin Marinas
Quoted 3 lines · Steve Capper
Rather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.
That's one thing I don't like about hugetlb, casting pmds to ptes. Do we actually save much in terms of code clean-up by doing this?
Hi Catalin,
At the moment we have ptes and hugetlb ptes behaving in the same way so they both make use of PTE_WRITE. I thought it would be more logical to tie the THP code to the pte manipulation functions too rather than duplicate the PTE_WRITE logic in a separate PMD_WRITE. This makes future pte changes apply to THP too automatically (along with hugetlb).
Cheers,
Signature · 1 line
Steve
Catalin Marinas
maintainerPATCH 2/3 arm64: mm: Route pmd thp functions through pte equivalents
Quoted 6 lines · Steve Capper
Quoted 5 lines · Catalin Marinas
Quoted 3 lines · Steve Capper
Rather than have separate hugetlb and transparent huge page pmd manipulation functions, re-wire our thp functions to simply call the pte equivalents.
Quoted 8 lines
Quoted 2 lines
That's one thing I don't like about hugetlb, casting pmds to ptes. Do we actually save much in terms of code clean-up by doing this?
At the moment we have ptes and hugetlb ptes behaving in the same way so they both make use of
PTE_WRITE. I thought it would be more logical to tie the THP code to the pte manipulation functions too rather than duplicate thePTE_WRITElogic in a separatePMD_WRITE. This makes future pte changes apply to THP too automatically (along with hugetlb).
OK, let's go with this, it's difficult to change hugetlb the other way.
BTW, powerpc and tile heavily simplify this using macros and pte_pmd/pmd_pte macros.
Signature · 1 line
Catalin
PATCH 3/3 arm64: mm: Correct definition of pmd_mknotpresent
In reply to Steve Capperpmd_mknotpresent currently creates a faulting pmd by clearing the valid bit. Unfortunately pmd_present(.) will report such pmds as being present!
This patch re-defines pmd_mknotpresent such that __pmd(0) is returned instead.
--- arch/arm64/include/asm/pgtable.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
arch/arm64/include/asm/pgtable.h +1 -2
@@ -291,8 +291,7 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd)static inline pmd_t pmd_mknotpresent (pmd_t pmd) { -pmd_val (pmd) &= ~PMD_TYPE_MASK ; -return pmd; +return __pmd (0 ); }#define pmd_mkhuge (pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))
Signature · 1 line
1.8.1.4
cover arm64: Use pte manipulation functions for THP
In reply to Steve CapperQuoted 7 lines · Steve Capper
This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new
PTE_WRITElogic, and provides better parity with the HugeTLB code (which already uses the pte functions).Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.
Hi,
Does this series look reasonable?
Thanks,
Signature · 1 line
Steve
Quoted 12 lines
Steve Capper (3):
arm64: mm: RemovePMD_BIT_FUNCmacro
arm64: mm: Route pmd thp functions through pte equivalents
arm64: mm: Correct definition ofpmd_mknotpresentarch/arm64/include/asm/pgtable.h | 91 +++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 29 deletions(-)Signature · 1 line
1.8.1.4
cover arm64: Use pte manipulation functions for THP
Quoted 11 lines · Steve Capper
Quoted 7 lines · Steve Capper
This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new
PTE_WRITElogic, and provides better parity with the HugeTLB code (which already uses the pte functions).Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.
Hi,
Does this series look reasonable?
I was waiting for the corresponding arch/arm/ changes.
Will
Catalin Marinas
maintainercover arm64: Use pte manipulation functions for THP
Quoted 7 lines · Will Deacon
Quoted 6 lines · Steve Capper
Quoted 5 lines · Steve Capper
This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new
PTE_WRITElogic, and provides better parity with the HugeTLB code (which already uses the pte functions).
Quoted 6 lines
Quoted 4 lines
Quoted 2 lines
Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.
Does this series look reasonable?
I was waiting for the corresponding
arch/arm/changes.
The arch/arm/ code doesn't have the PTE_WRITE changes. It's a bit more work here because of the classic MMU (but I hope Steve will get there ;).
Signature · 1 line
Catalin
cover arm64: Use pte manipulation functions for THP
Quoted 11 lines · Catalin Marinas
Quoted 10 lines · Will Deacon
Quoted 9 lines · Steve Capper
Quoted 7 lines · Steve Capper
This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new
PTE_WRITElogic, and provides better parity with the HugeTLB code (which already uses the pte functions).Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.
Quoted 6 lines
Quoted 3 lines
Quoted 1 line
Does this series look reasonable?
I was waiting for the corresponding
arch/arm/changes.The
arch/arm/code doesn't have thePTE_WRITEchanges. It's a bit more work here because of the classic MMU (but I hope Steve will get there ;).
Indeed, but I'm wary of divergence in the mm code between the two architectures and I'd much rather we try to keep them in sync, rather than improve/bug fix in one before porting the whole lot over.
Will
cover arm64: Use pte manipulation functions for THP
Quoted 14 lines · Will Deacon
Quoted 13 lines · Catalin Marinas
Quoted 12 lines · Will Deacon
Quoted 10 lines · Steve Capper
Quoted 7 lines · Steve Capper
This series replaces the Transparent HugePage pmd manipulation functions with calls to the standard pte functions. This allows the THP code to take advantage of the new
PTE_WRITElogic, and provides better parity with the HugeTLB code (which already uses the pte functions).Testing was done on the Fast Model with LTP THP tests, and the 3.14-rc1 kernel was used.
Does this series look reasonable?
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
I was waiting for the corresponding
arch/arm/changes.The
arch/arm/code doesn't have thePTE_WRITEchanges. It's a bit more work here because of the classic MMU (but I hope Steve will get there ;).Indeed, but I'm wary of divergence in the mm code between the two architectures and I'd much rather we try to keep them in sync, rather than improve/bug fix in one before porting the whole lot over.
I do think we need the same behaviour for 3-level code on ARM and for ARM64.
The PTE_WRITE changes for ARM64 allow one to distinguish between writable clean ptes and read only ptes which fixes a very subtle problem with huge pages (and maybe some other problems). This problem is still present in the ARM 3-level code.
The ARM 2-level code stores a separate linux/hardware pte and it is possible to distinguish between writable clean ptes and read only ptes.
For ARM, I would like to split out some of the pte logic and page permissions from pgtable.h into pgtable-[23]level.h. Then introduce PTE_WRITE for the 3-level code on ARM.
Russell,
I have noticed the following for ARM:
36bb94b ARM: pgtable: provide RDONLY page table bit rather than WRITE bit
Are you happy for me to re-introduce PTE_WRITE for the 3-level code only, and keep L_PTE_RDONLY for the 2-level code?
Thanks,
Signature · 1 line
Steve
cover arm64: Use pte manipulation functions for THP
Quoted 4 lines · Steve Capper
I have noticed the following for ARM:
36bb94b ARM: pgtable: provide RDONLY page table bit rather than WRITE bit
Are you happy for me to re-introducePTE_WRITEfor the 3-level code only, and keepL_PTE_RDONLYfor the 2-level code?
Sigh, yet another cleanup reverted... If you must.
Signature · 3 lines
FTTC broadband for 0.8mile line: 5.8Mbps down 500kbps up. Estimation in database were 13.1 to 19Mbit for a good line, about 7.5+ for a bad. Estimate before purchase was "up to 13.2Mbit".
Catalin Marinas
maintainercover arm64: Use pte manipulation functions for THP
In reply to Steve CapperQuoted 4 lines · Steve Capper
I have noticed the following for ARM:
36bb94b ARM: pgtable: provide RDONLY page table bit rather than WRITE bit
Are you happy for me to re-introducePTE_WRITEfor the 3-level code only, and keepL_PTE_RDONLYfor the 2-level code?
But don't we create more confusion by differentiating between 2-level and 3-level? What about L_PTE_WRITE in all cases as pure software bit and PTE_RDONLY (without L_) defined just for LPAE.
Signature · 1 line
Catalin