ARM: mm: HugeTLB + THP support for non-LPAE
Subsystems (2)
cover ARM: mm: HugeTLB + THP support for non-LPAE
Apologies for the noise, I resent this series but neglected to place the word "PATCH" in the subject lines of the patches thus the patches got blocked by the list server. Here is another go at a resend.
---
The following patches bring both HugeTLB support and Transparent HugePage (THP) support to ARM for 2 levels of paging (i.e. without LPAE).
This code has been tested on an Arndale board (Exynos 5250), and is based on 3.11-rc1, with the hugepage simplification patch:
http://lists.infradead.org/pipermail/linux-arm-kernel/2013-July/184117.html
HugeTLB and THP support for LPAE has already been merged in 3.11-rc1.
Hugepages can give nice performance boosts to workloads that put pressure on the TLBs. I've observed uplifts of ~5% to some tasks just by enabling hugepages via the libhugetlbfs tools. Other people have observed decent performance boosts when huge pages are enabled:
http://lists.infradead.org/pipermail/linux-arm-kernel/2013-February/148835.html
I would appreciate any discussion on these patches, as there are people who have an interest in having huge page support in non-LPAE kernels.
Steve Capper (2):
ARM: mm: HugeTLB support for non-LPAE systems.
ARM: mm: Transparent huge page support for non-LPAE systems.
arch/arm/Kconfig | 4 +- arch/arm/include/asm/hugetlb-2level.h | 126 +++++++++++++++++++++++++ arch/arm/include/asm/hugetlb.h | 4 + arch/arm/include/asm/pgtable-2level.h | 170 ++++++++++++++++++++++++++++++++++ arch/arm/include/asm/pgtable-3level.h | 6 ++ arch/arm/include/asm/pgtable.h | 7 +- arch/arm/include/asm/tlb.h | 10 +- arch/arm/kernel/head.S | 10 +- arch/arm/mm/fault.c | 13 --- arch/arm/mm/fsr-2level.c | 4 +- arch/arm/mm/hugetlbpage.c | 2 +- arch/arm/mm/mmu.c | 27 ++++++ 12 files changed, 360 insertions(+), 23 deletions(-) create mode 100644 arch/arm/include/asm/hugetlb-2level.h
Signature · 1 line
1.8.1.4
RFC PATCH 1/2 ARM: mm: HugeTLB support for non-LPAE systems.
Based on Bill Carson's HugeTLB patch, with the big difference being in the way PTEs are passed back to the memory manager. Rather than store a "Linux Huge PTE" separately; we make one up on the fly in huge_ptep_get. Also rather than consider 16M supersections, we focus solely on 2x1M sections.
To construct a huge PTE on the fly we need additional information (such as the accessed flag and dirty bit) which we choose to store in the domain bits of the short section descriptor. In order to use these domain bits for storage, we need to make ourselves a client for all 16 domains and this is done in head.S.
Storing extra information in the domain bits also makes it a lot easier to implement Transparent Huge Pages, and some of the code in pgtable-2level.h is arranged to facilitate THP support in a later patch.
Non-LPAE HugeTLB pages are incompatible with the huge page migration code (enabled when CONFIG_MEMORY_FAILURE is selected) as that code dereferences PTEs directly, rather than calling huge_ptep_get and set_huge_pte_at.
--- arch/arm/Kconfig | 2 +- arch/arm/include/asm/hugetlb-2level.h | 126 ++++++++++++++++++++++++++++++++++ arch/arm/include/asm/hugetlb.h | 4 ++ arch/arm/include/asm/pgtable-2level.h | 123 +++++++++++++++++++++++++++++++++ arch/arm/include/asm/pgtable-3level.h | 5 ++ arch/arm/include/asm/pgtable.h | 5 ++ arch/arm/include/asm/tlb.h | 10 ++- arch/arm/kernel/head.S | 10 ++- arch/arm/mm/fault.c | 13 ---- arch/arm/mm/fsr-2level.c | 4 +- arch/arm/mm/hugetlbpage.c | 2 +- arch/arm/mm/mmu.c | 27 ++++++++ 12 files changed, 311 insertions(+), 20 deletions(-) create mode 100644 arch/arm/include/asm/hugetlb-2level.h
arch/arm/Kconfig +1 -1
@@ -1751,7 +1751,7 @@ config HW_PERF_EVENTS config SYS_SUPPORTS_HUGETLBFS def_bool y - depends on ARM_LPAE + depends on ARM_LPAE || (!CPU_USE_DOMAINS && !MEMORY_FAILURE) config HAVE_ARCH_TRANSPARENT_HUGEPAGE def_bool y
arch/arm/include/asm/hugetlb-2level.h new file +126 -0
126 lines not on this page · raw message
arch/arm/include/asm/hugetlb.h +4 -0
@@ -25,7 +25,11 @@#include <asm/page.h> #include <asm-generic/hugetlb.h> +#ifdef CONFIG_ARM_LPAE #include <asm/hugetlb-3level.h> +#else +#include <asm/hugetlb-2level.h> +#endif static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end,
arch/arm/include/asm/pgtable-2level.h +123 -0
135 lines not on this page · raw message
arch/arm/include/asm/pgtable-3level.h +5 -0
@@ -153,6 +153,11 @@flush_pmd_entry (pudp); \ }while (0 ) +static inline int pmd_large(pmd_t pmd) +{ +return pmd_val (pmd) && !(pmd_val (pmd) &PMD_TABLE_BIT ); +} +static inline pmd_t *pud_page_vaddr (pud_t pud) {return __va (pud_val (pud) &PHYS_MASK & (s32 )PAGE_MASK );
arch/arm/include/asm/pgtable.h +5 -0
@@ -220,6 +220,7 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)#define pte_dirty (pte) (pte_val(pte) & L_PTE_DIRTY)#define pte_young (pte) (pte_val(pte) & L_PTE_YOUNG)#define pte_exec (pte) (!(pte_val(pte) & L_PTE_XN)) +#define pte_protnone (pte) (pte_val(pte) & L_PTE_NONE)#define pte_special (pte) (0)#define pte_present_user (pte) (pte_present(pte) && (pte_val(pte) & L_PTE_USER))
@@ -254,6 +255,10 @@ PTE_BIT_FUNC(mkclean, &= ~L_PTE_DIRTY); PTE_BIT_FUNC(mkdirty, |= L_PTE_DIRTY); PTE_BIT_FUNC(mkold, &= ~L_PTE_YOUNG); PTE_BIT_FUNC(mkyoung, |= L_PTE_YOUNG); +PTE_BIT_FUNC(mkexec, &= ~L_PTE_XN); +PTE_BIT_FUNC(mknexec, |= L_PTE_XN); +PTE_BIT_FUNC(mkprotnone,|= L_PTE_NONE); +PTE_BIT_FUNC(rmprotnone,&= ~L_PTE_NONE);static inline pte_t pte_mkspecial (pte_t pte) {return pte; }
arch/arm/include/asm/tlb.h +8 -2
@@ -80,10 +80,16 @@ static inline void tlb_flush(struct mmu_gather *tlb)static inline void tlb_add_flush (struct mmu_gather *tlb,unsigned long addr) {if (!tlb->fullmm) { +unsigned long size =PAGE_SIZE ; +if (addr < tlb->range_start) tlb->range_start = addr; -if (addr +PAGE_SIZE > tlb->range_end) - tlb->range_end = addr +PAGE_SIZE ; + +if (tlb->vma &&is_vm_hugetlb_page (tlb->vma)) + size =HPAGE_SIZE ; + +if (addr + size > tlb->range_end) + tlb->range_end = addr + size; } }
arch/arm/kernel/head.S +9 -1
@@ -429,13 +429,21 @@ __enable_mmu: bic r0, r0, #CR_I #endif #ifndef CONFIG_ARM_LPAE +#ifndef CONFIG_SYS_SUPPORTS_HUGETLBFS mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ domain_val(DOMAIN_IO, DOMAIN_CLIENT)) +#else + @ set ourselves as the client in all domains + @ this allows us to then use the 4 domain bits in the + @ section descriptors in our transparent huge pages + ldr r5, =0x55555555 +#endif /* CONFIG_SYS_SUPPORTS_HUGETLBFS */ + mcr p15, 0, r5, c3, c0, 0 @ load domain access register mcr p15, 0, r4, c2, c0, 0 @ load page table pointer -#endif +#endif /* CONFIG_ARM_LPAE */ b __turn_mmu_on ENDPROC(__enable_mmu)
arch/arm/mm/fault.c +0 -13
@@ -488,19 +488,6 @@ do_translation_fault(unsigned long addr, unsigned int fsr,#endif /* CONFIG_MMU */ /* -* Some section permission faults need to be handled gracefully. -* They can happen due to a __{get,put}_user during an oops. -*/ -#ifndef CONFIG_ARM_LPAE -static int -do_sect_fault (unsigned long addr,unsigned int fsr,struct pt_regs *regs) -{ -do_bad_area (addr, fsr, regs); -return 0 ; -} -#endif /* CONFIG_ARM_LPAE */ - -/* * This abort handler always returns "fault". */ static int
arch/arm/mm/fsr-2level.c +2 -2
@@ -16,7 +16,7 @@ static struct fsr_info fsr_info[] = { { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },/*
@@ -56,7 +56,7 @@ static struct fsr_info ifsr_info[] = { { do_bad, SIGBUS, 0, "unknown 10" }, { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, { do_bad, SIGBUS, 0, "external abort on translation" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, { do_bad, SIGBUS, 0, "unknown 16" },
arch/arm/mm/hugetlbpage.c +1 -1
@@ -54,5 +54,5 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)int pmd_huge (pmd_t pmd) { -return pmd_val (pmd) && !(pmd_val (pmd) &PMD_TABLE_BIT ); +return pmd_large (pmd); }
arch/arm/mm/mmu.c +27 -0
@@ -336,6 +336,21 @@ const struct mem_type *get_mem_type(unsigned int type)EXPORT_SYMBOL (get_mem_type);/* +* If the system supports huge pages and we are running with short descriptors, +* then compute the pmd and linux pte prot values for a huge page. +* +* These values are used by both the HugeTLB and THP code. +*/ +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS ) && !defined(CONFIG_ARM_LPAE ) +pmdval_t arm_hugepmdprotval; +EXPORT_SYMBOL (arm_hugepmdprotval); + +pteval_t arm_hugepteprotval; +EXPORT_SYMBOL (arm_hugepteprotval); +#endif + + +/* * Adjust the PMD section entries according to the CPU in use. */ static void __init build_mem_type_table (void )
@@ -566,6 +581,18 @@ static void __init build_mem_type_table(void)if (t->prot_sect) t->prot_sect |=PMD_DOMAIN (t->domain); } + +#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS ) && !defined(CONFIG_ARM_LPAE ) +/* +* we assume all huge pages are user pages and that hardware access +* flag updates are disabled (i.e. SCTLR.AFE == 0b). +*/ + arm_hugepteprotval = mem_types[MT_MEMORY ].prot_pte |L_PTE_USER |L_PTE_VALID ; + + arm_hugepmdprotval = mem_types[MT_MEMORY ].prot_sect |PMD_SECT_AP_READ + | PMD_SECT_nG; +#endif + }#ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
Signature · 1 line
1.8.1.4
RFC PATCH 2/2 ARM: mm: Transparent huge page support for non-LPAE systems.
In reply to Steve CapperMuch of the required code for THP has been implemented in the earlier non-LPAE HugeTLB patch.
One more domain bit is used (to store whether or not the THP is splitting).
Some THP helper functions are defined; and we have to re-define pmd_page such that it distinguishes between page tables and sections.
--- arch/arm/Kconfig | 2 +- arch/arm/include/asm/pgtable-2level.h | 47 +++++++++++++++++++++++++++++++++++ arch/arm/include/asm/pgtable-3level.h | 1 + arch/arm/include/asm/pgtable.h | 2 -- 4 files changed, 49 insertions(+), 3 deletions(-)
arch/arm/Kconfig +1 -1
@@ -1755,7 +1755,7 @@ config SYS_SUPPORTS_HUGETLBFS config HAVE_ARCH_TRANSPARENT_HUGEPAGE def_bool y - depends on ARM_LPAE + depends on SYS_SUPPORTS_HUGETLBFS config ARCH_WANT_GENERAL_HUGETLB def_bool y
arch/arm/include/asm/pgtable-2level.h +47 -0
59 lines not on this page · raw message
arch/arm/include/asm/pgtable-3level.h +1 -0
@@ -234,6 +234,7 @@ PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF);/* represent a notpresent pmd by zero, this is used by pmdp_invalidate */ #define pmd_mknotpresent (pmd) (__pmd(0)) +#define pmd_page (pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))static inline pmd_t pmd_modify (pmd_t pmd,pgprot_t newprot) {
arch/arm/include/asm/pgtable.h +0 -2
@@ -189,8 +189,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)return __va (pmd_val (pmd) &PHYS_MASK & (s32 )PAGE_MASK ); } -#define pmd_page (pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) -#ifndef CONFIG_HIGHPTE #define __pte_map (pmd) pmd_page_vaddr(*(pmd))#define __pte_unmap (pte) do { } while (0)
Signature · 1 line
1.8.1.4