Skip to contentSkip to conversation

ARM: mm: HugeTLB + THP support for non-LPAE

RFC New Steve Capper 1 participant 3 messages Oct 9, 2013

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover ARM: mm: HugeTLB + THP support for non-LPAE

Apologies for the noise, I resent this series but neglected to place the word "PATCH" in the subject lines of the patches thus the patches got blocked by the list server. Here is another go at a resend.
---

The following patches bring both HugeTLB support and Transparent HugePage (THP) support to ARM for 2 levels of paging (i.e. without LPAE).

This code has been tested on an Arndale board (Exynos 5250), and is based on 3.11-rc1, with the hugepage simplification patch:
http://lists.infradead.org/pipermail/linux-arm-kernel/2013-July/184117.html

HugeTLB and THP support for LPAE has already been merged in 3.11-rc1.

Hugepages can give nice performance boosts to workloads that put pressure on the TLBs. I've observed uplifts of ~5% to some tasks just by enabling hugepages via the libhugetlbfs tools. Other people have observed decent performance boosts when huge pages are enabled:
http://lists.infradead.org/pipermail/linux-arm-kernel/2013-February/148835.html

I would appreciate any discussion on these patches, as there are people who have an interest in having huge page support in non-LPAE kernels.

Steve Capper (2):
ARM: mm: HugeTLB support for non-LPAE systems.
ARM: mm: Transparent huge page support for non-LPAE systems.

 arch/arm/Kconfig                      |   4 +-
 arch/arm/include/asm/hugetlb-2level.h | 126 +++++++++++++++++++++++++
 arch/arm/include/asm/hugetlb.h        |   4 +
 arch/arm/include/asm/pgtable-2level.h | 170 ++++++++++++++++++++++++++++++++++
 arch/arm/include/asm/pgtable-3level.h |   6 ++
 arch/arm/include/asm/pgtable.h        |   7 +-
 arch/arm/include/asm/tlb.h            |  10 +-
 arch/arm/kernel/head.S                |  10 +-
 arch/arm/mm/fault.c                   |  13 ---
 arch/arm/mm/fsr-2level.c              |   4 +-
 arch/arm/mm/hugetlbpage.c             |   2 +-
 arch/arm/mm/mmu.c                     |  27 ++++++
 12 files changed, 360 insertions(+), 23 deletions(-)
 create mode 100644 arch/arm/include/asm/hugetlb-2level.h
Signature · 1 line
1.8.1.4
#1 lore raw

RFC PATCH 1/2 ARM: mm: HugeTLB support for non-LPAE systems.

Based on Bill Carson's HugeTLB patch, with the big difference being in the way PTEs are passed back to the memory manager. Rather than store a "Linux Huge PTE" separately; we make one up on the fly in huge_ptep_get. Also rather than consider 16M supersections, we focus solely on 2x1M sections.

To construct a huge PTE on the fly we need additional information (such as the accessed flag and dirty bit) which we choose to store in the domain bits of the short section descriptor. In order to use these domain bits for storage, we need to make ourselves a client for all 16 domains and this is done in head.S.

Storing extra information in the domain bits also makes it a lot easier to implement Transparent Huge Pages, and some of the code in pgtable-2level.h is arranged to facilitate THP support in a later patch.

Non-LPAE HugeTLB pages are incompatible with the huge page migration code (enabled when CONFIG_MEMORY_FAILURE is selected) as that code dereferences PTEs directly, rather than calling huge_ptep_get and set_huge_pte_at.

---
 arch/arm/Kconfig                      |   2 +-
 arch/arm/include/asm/hugetlb-2level.h | 126 ++++++++++++++++++++++++++++++++++
 arch/arm/include/asm/hugetlb.h        |   4 ++
 arch/arm/include/asm/pgtable-2level.h | 123 +++++++++++++++++++++++++++++++++
 arch/arm/include/asm/pgtable-3level.h |   5 ++
 arch/arm/include/asm/pgtable.h        |   5 ++
 arch/arm/include/asm/tlb.h            |  10 ++-
 arch/arm/kernel/head.S                |  10 ++-
 arch/arm/mm/fault.c                   |  13 ----
 arch/arm/mm/fsr-2level.c              |   4 +-
 arch/arm/mm/hugetlbpage.c             |   2 +-
 arch/arm/mm/mmu.c                     |  27 ++++++++
 12 files changed, 311 insertions(+), 20 deletions(-)
 create mode 100644 arch/arm/include/asm/hugetlb-2level.h
arch/arm/Kconfig +1 -1
index d0fd5bf..fe6eeae 100644
@@ -1751,7 +1751,7 @@ config HW_PERF_EVENTS
 
 config SYS_SUPPORTS_HUGETLBFS
        def_bool y
-       depends on ARM_LPAE
+       depends on ARM_LPAE || (!CPU_USE_DOMAINS && !MEMORY_FAILURE)
 
 config HAVE_ARCH_TRANSPARENT_HUGEPAGE
        def_bool y
arch/arm/include/asm/hugetlb-2level.h new file +126 -0

126 lines not on this page · raw message

arch/arm/include/asm/hugetlb.h +4 -0
index 1f1b1cd..6c0372a 100644
@@ -25,7 +25,11 @@
 #include <asm/page.h>
 #include <asm-generic/hugetlb.h>
 
+#ifdef CONFIG_ARM_LPAE
 #include <asm/hugetlb-3level.h>
+#else
+#include <asm/hugetlb-2level.h>
+#endif
 
 static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb,
 					  unsigned long addr, unsigned long end,
arch/arm/include/asm/pgtable-2level.h +123 -0

135 lines not on this page · raw message

arch/arm/include/asm/pgtable-3level.h +5 -0
index 5689c18..67a0e06 100644
@@ -153,6 +153,11 @@
 		flush_pmd_entry(pudp);	\
 	} while (0)
 
+static inline int pmd_large(pmd_t pmd)
+{
+	return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
+}
+
 static inline pmd_t *pud_page_vaddr(pud_t pud)
 {
 	return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
arch/arm/include/asm/pgtable.h +5 -0
index 04aeb02..cf77a59 100644
@@ -220,6 +220,7 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 #define pte_dirty(pte)		(pte_val(pte) & L_PTE_DIRTY)
 #define pte_young(pte)		(pte_val(pte) & L_PTE_YOUNG)
 #define pte_exec(pte)		(!(pte_val(pte) & L_PTE_XN))
+#define pte_protnone(pte)	(pte_val(pte) & L_PTE_NONE)
 #define pte_special(pte)	(0)
 
 #define pte_present_user(pte)  (pte_present(pte) && (pte_val(pte) & L_PTE_USER))
@@ -254,6 +255,10 @@ PTE_BIT_FUNC(mkclean,   &= ~L_PTE_DIRTY);
 PTE_BIT_FUNC(mkdirty,   |= L_PTE_DIRTY);
 PTE_BIT_FUNC(mkold,     &= ~L_PTE_YOUNG);
 PTE_BIT_FUNC(mkyoung,   |= L_PTE_YOUNG);
+PTE_BIT_FUNC(mkexec,	&= ~L_PTE_XN);
+PTE_BIT_FUNC(mknexec,	|= L_PTE_XN);
+PTE_BIT_FUNC(mkprotnone,|= L_PTE_NONE);
+PTE_BIT_FUNC(rmprotnone,&= ~L_PTE_NONE);
 
 static inline pte_t pte_mkspecial(pte_t pte) { return pte; }
 
arch/arm/include/asm/tlb.h +8 -2
index 46e7cfb..ddb77e5 100644
@@ -80,10 +80,16 @@ static inline void tlb_flush(struct mmu_gather *tlb)
 static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
 {
 	if (!tlb->fullmm) {
+		unsigned long size = PAGE_SIZE;
+
 		if (addr < tlb->range_start)
 			tlb->range_start = addr;
-		if (addr + PAGE_SIZE > tlb->range_end)
-			tlb->range_end = addr + PAGE_SIZE;
+
+		if (tlb->vma && is_vm_hugetlb_page(tlb->vma))
+			size = HPAGE_SIZE;
+
+		if (addr + size > tlb->range_end)
+			tlb->range_end = addr + size;
 	}
 }
 
arch/arm/kernel/head.S +9 -1
index 45e8935..fb1df2f 100644
@@ -429,13 +429,21 @@ __enable_mmu:
 	bic	r0, r0, #CR_I
 #endif
 #ifndef CONFIG_ARM_LPAE
+#ifndef	CONFIG_SYS_SUPPORTS_HUGETLBFS
 	mov	r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \
 		      domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \
 		      domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \
 		      domain_val(DOMAIN_IO, DOMAIN_CLIENT))
+#else
+	@ set ourselves as the client in all domains
+	@ this allows us to then use the 4 domain bits in the
+	@ section descriptors in our transparent huge pages
+	ldr	r5, =0x55555555
+#endif /* CONFIG_SYS_SUPPORTS_HUGETLBFS */
+
 	mcr	p15, 0, r5, c3, c0, 0		@ load domain access register
 	mcr	p15, 0, r4, c2, c0, 0		@ load page table pointer
-#endif
+#endif /* CONFIG_ARM_LPAE */
 	b	__turn_mmu_on
 ENDPROC(__enable_mmu)
 
arch/arm/mm/fault.c +0 -13
index c97f794..95d53f9 100644
@@ -488,19 +488,6 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
 #endif					/* CONFIG_MMU */
 
 /*
- * Some section permission faults need to be handled gracefully.
- * They can happen due to a __{get,put}_user during an oops.
- */
-#ifndef CONFIG_ARM_LPAE
-static int
-do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
-{
-	do_bad_area(addr, fsr, regs);
-	return 0;
-}
-#endif /* CONFIG_ARM_LPAE */
-
-/*
  * This abort handler always returns "fault".
  */
 static int
arch/arm/mm/fsr-2level.c +2 -2
index 18ca74c..c1a2afc 100644
@@ -16,7 +16,7 @@ static struct fsr_info fsr_info[] = {
 	{ do_bad,		SIGBUS,	 0,		"external abort on non-linefetch"  },
 	{ do_bad,		SIGSEGV, SEGV_ACCERR,	"page domain fault"		   },
 	{ do_bad,		SIGBUS,	 0,		"external abort on translation"	   },
-	{ do_sect_fault,	SIGSEGV, SEGV_ACCERR,	"section permission fault"	   },
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"section permission fault"	   },
 	{ do_bad,		SIGBUS,	 0,		"external abort on translation"	   },
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"page permission fault"		   },
 	/*
@@ -56,7 +56,7 @@ static struct fsr_info ifsr_info[] = {
 	{ do_bad,		SIGBUS,  0,		"unknown 10"			   },
 	{ do_bad,		SIGSEGV, SEGV_ACCERR,	"page domain fault"		   },
 	{ do_bad,		SIGBUS,	 0,		"external abort on translation"	   },
-	{ do_sect_fault,	SIGSEGV, SEGV_ACCERR,	"section permission fault"	   },
+	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"section permission fault"	   },
 	{ do_bad,		SIGBUS,	 0,		"external abort on translation"	   },
 	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"page permission fault"		   },
 	{ do_bad,		SIGBUS,  0,		"unknown 16"			   },
arch/arm/mm/hugetlbpage.c +1 -1
index 66781bf..5b63ec6 100644
@@ -54,5 +54,5 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
 
 int pmd_huge(pmd_t pmd)
 {
-	return pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT);
+	return pmd_large(pmd);
 }
arch/arm/mm/mmu.c +27 -0
index 4f56617..e39dc0b 100644
@@ -336,6 +336,21 @@ const struct mem_type *get_mem_type(unsigned int type)
 EXPORT_SYMBOL(get_mem_type);
 
 /*
+ * If the system supports huge pages and we are running with short descriptors,
+ * then compute the pmd and linux pte prot values for a huge page.
+ *
+ * These values are used by both the HugeTLB and THP code.
+ */
+#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE)
+pmdval_t arm_hugepmdprotval;
+EXPORT_SYMBOL(arm_hugepmdprotval);
+
+pteval_t arm_hugepteprotval;
+EXPORT_SYMBOL(arm_hugepteprotval);
+#endif
+
+
+/*
  * Adjust the PMD section entries according to the CPU in use.
  */
 static void __init build_mem_type_table(void)
@@ -566,6 +581,18 @@ static void __init build_mem_type_table(void)
 		if (t->prot_sect)
 			t->prot_sect |= PMD_DOMAIN(t->domain);
 	}
+
+#if defined(CONFIG_SYS_SUPPORTS_HUGETLBFS) && !defined(CONFIG_ARM_LPAE)
+	/*
+	 * we assume all huge pages are user pages and that hardware access
+	 * flag updates are disabled (i.e. SCTLR.AFE == 0b).
+	 */
+	arm_hugepteprotval = mem_types[MT_MEMORY].prot_pte | L_PTE_USER | L_PTE_VALID;
+
+	arm_hugepmdprotval = mem_types[MT_MEMORY].prot_sect | PMD_SECT_AP_READ
+				| PMD_SECT_nG;
+#endif
+
 }
 
 #ifdef CONFIG_ARM_DMA_MEM_BUFFERABLE
Signature · 1 line
1.8.1.4
#2 lore raw

RFC PATCH 2/2 ARM: mm: Transparent huge page support for non-LPAE systems.

In reply to Steve Capper

Much of the required code for THP has been implemented in the earlier non-LPAE HugeTLB patch.

One more domain bit is used (to store whether or not the THP is splitting).

Some THP helper functions are defined; and we have to re-define pmd_page such that it distinguishes between page tables and sections.

---
 arch/arm/Kconfig                      |  2 +-
 arch/arm/include/asm/pgtable-2level.h | 47 +++++++++++++++++++++++++++++++++++
 arch/arm/include/asm/pgtable-3level.h |  1 +
 arch/arm/include/asm/pgtable.h        |  2 --
 4 files changed, 49 insertions(+), 3 deletions(-)
arch/arm/Kconfig +1 -1
index fe6eeae..6b53969 100644
@@ -1755,7 +1755,7 @@ config SYS_SUPPORTS_HUGETLBFS
 
 config HAVE_ARCH_TRANSPARENT_HUGEPAGE
        def_bool y
-       depends on ARM_LPAE
+       depends on SYS_SUPPORTS_HUGETLBFS
 
 config ARCH_WANT_GENERAL_HUGETLB
 	def_bool y
arch/arm/include/asm/pgtable-2level.h +47 -0

59 lines not on this page · raw message

arch/arm/include/asm/pgtable-3level.h +1 -0
index 67a0e06..9c38f29 100644
@@ -234,6 +234,7 @@ PMD_BIT_FUNC(mkyoung,   |= PMD_SECT_AF);
 
 /* represent a notpresent pmd by zero, this is used by pmdp_invalidate */
 #define pmd_mknotpresent(pmd)	(__pmd(0))
+#define pmd_page(pmd)           pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
 
 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
 {
arch/arm/include/asm/pgtable.h +0 -2
index cf77a59..66a1417 100644
@@ -189,8 +189,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 	return __va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK);
 }
 
-#define pmd_page(pmd)		pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
-
 #ifndef CONFIG_HIGHPTE
 #define __pte_map(pmd)		pmd_page_vaddr(*(pmd))
 #define __pte_unmap(pte)	do { } while (0)
Signature · 1 line
1.8.1.4