arm64: mm: Implement 4 levels of translation tables
Subsystems (3)
PATCH v6 5/7 arm64: mm: Implement 4 levels of translation tables
This patch implements 4 levels of translation tables since 3 levels of page tables with 4KB pages cannot support 40-bit physical address space described in [1] due to the following issue.
It is a restriction that kernel logical memory map with 4KB + 3 levels (0xffffffc000000000-0xffffffffffffffff) cannot cover RAM region from 544GB to 1024GB in [1]. Specifically, ARM64 kernel fails to create mapping for this region in map_mem function since __phys_to_virt for this region reaches to address overflow.
If SoC design follows the document, [1], over 32GB RAM would be placed from 544GB. Even 64GB system is supposed to use the region from 544GB to 576GB for only 32GB RAM. Naturally, it would reach to enable 4 levels of page tables to avoid hacking __virt_to_phys and __phys_to_virt.
However, it is recommended 4 levels of page table should be only enabled if memory map is too sparse or there is about 512GB RAM.
References
----------
[1]: Principles of ARM Memory Maps, White Paper, Issue C
Cc Catalin Marinas <[email protected]>
Cc Steve Capper <[email protected]>
Signed-off-by Jungseok Lee <[email protected]>
Reviewed-by Sungjinn Chung <[email protected]>
Acked-by Kukjin Kim <[email protected]>
Reviewed-by Christoffer Dall <[email protected]>
Reviewed-by Steve Capper <[email protected]>
--- arch/arm64/Kconfig | 8 +++++ arch/arm64/include/asm/memblock.h | 6 ++++ arch/arm64/include/asm/page.h | 4 ++- arch/arm64/include/asm/pgalloc.h | 20 ++++++++++++ arch/arm64/include/asm/pgtable-hwdef.h | 6 ++-- arch/arm64/include/asm/pgtable.h | 45 ++++++++++++++++++++++++++ arch/arm64/include/asm/tlb.h | 9 ++++++ arch/arm64/kernel/head.S | 54 +++++++++++++++++++++++++------- arch/arm64/kernel/traps.c | 5 +++ arch/arm64/mm/fault.c | 1 + arch/arm64/mm/mmu.c | 16 +++++++--- 11 files changed, 156 insertions(+), 18 deletions(-)
arch/arm64/Kconfig +8 -0
@@ -182,12 +182,17 @@ config ARM64_VA_BITS_42 bool "42-bit" depends on ARM64_64K_PAGES +config ARM64_VA_BITS_48 + bool "48-bit" + depends on ARM64_4K_PAGES + endchoice config ARM64_VA_BITS int default 39 if ARM64_VA_BITS_39 default 42 if ARM64_VA_BITS_42 + default 48 if ARM64_VA_BITS_48 config ARM64_2_LEVELS def_bool y if ARM64_64K_PAGES && ARM64_VA_BITS_42
@@ -195,6 +200,9 @@ config ARM64_2_LEVELS config ARM64_3_LEVELS def_bool y if ARM64_4K_PAGES && ARM64_VA_BITS_39 +config ARM64_4_LEVELS + def_bool y if ARM64_4K_PAGES && ARM64_VA_BITS_48 + config CPU_BIG_ENDIAN bool "Build big-endian kernel" help
arch/arm64/include/asm/memblock.h +6 -0
@@ -16,6 +16,12 @@#ifndef __ASM_MEMBLOCK_H#define __ASM_MEMBLOCK_H +#ifndef CONFIG_ARM64_4_LEVELS +#define MEMBLOCK_INITIAL_LIMIT PGDIR_SIZE +#else +#define MEMBLOCK_INITIAL_LIMIT PUD_SIZE +#endif +extern void arm64_memblock_init (void );#endif
arch/arm64/include/asm/page.h +3 -1
@@ -35,8 +35,10 @@#ifdef CONFIG_ARM64_2_LEVELS #include <asm/pgtable-2level-types.h> -#else +#elif defined(CONFIG_ARM64_3_LEVELS )#include <asm/pgtable-3level-types.h> +#else +#include <asm/pgtable-4level-types.h> #endif extern void __cpu_clear_user_page (void *p,unsigned long user);
arch/arm64/include/asm/pgalloc.h +20 -0
@@ -26,6 +26,26 @@#define check_pgt_cache () do { } while (0) +#ifdef CONFIG_ARM64_4_LEVELS + +static inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) +{ +return (pud_t *)get_zeroed_page (GFP_KERNEL | __GFP_REPEAT); +} + +static inline void pud_free (struct mm_struct *mm,pud_t *pud) +{ +BUG_ON ((unsigned long )pud & (PAGE_SIZE -1 )); +free_page ((unsigned long )pud); +} + +static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,pud_t *pud) +{ +set_pgd (pgd,__pgd (__pa (pud) |PUD_TYPE_TABLE )); +} + +#endif /* CONFIG_ARM64_4_LEVELS */ +#ifndef CONFIG_ARM64_2_LEVELS static inline pmd_t *pmd_alloc_one (struct mm_struct *mm,unsigned long addr)
arch/arm64/include/asm/pgtable-hwdef.h +4 -2
@@ -18,8 +18,10 @@#ifdef CONFIG_ARM64_2_LEVELS #include <asm/pgtable-2level-hwdef.h> -#else +#elif defined(CONFIG_ARM64_3_LEVELS )#include <asm/pgtable-3level-hwdef.h> +#else +#include <asm/pgtable-4level-hwdef.h> #endif /*
@@ -27,7 +29,7 @@* * Level 1 descriptor (PUD). */ - +#define PUD_TYPE_TABLE (_AT(pudval_t, 3) << 0)#define PUD_TABLE_BIT (_AT(pgdval_t, 1) << 1)#define PUD_TYPE_MASK (_AT(pgdval_t, 3) << 0)#define PUD_TYPE_SECT (_AT(pgdval_t, 1) << 0)
arch/arm64/include/asm/pgtable.h +45 -0
84 lines not on this page · raw message
arch/arm64/include/asm/tlb.h +9 -0
@@ -100,6 +100,15 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, }#endif +#ifdef CONFIG_ARM64_4_LEVELS +static inline void __pud_free_tlb (struct mmu_gather *tlb,pud_t *pudp, +unsigned long addr) +{ +tlb_add_flush (tlb, addr); +tlb_remove_page (tlb,virt_to_page (pudp)); +} +#endif +static inline void __tlb_remove_pmd_tlb_entry (struct mmu_gather *tlb,pmd_t *pmdp,unsigned long address) {
arch/arm64/kernel/head.S +43 -11
96 lines not on this page · raw message
arch/arm64/kernel/traps.c +5 -0
@@ -339,6 +339,11 @@ void __pmd_error(const char *file, int line, unsigned long val)pr_crit ("%s:%d: bad pmd %016lx.\n" , file, line, val); } +void __pud_error (const char *file,int line,unsigned long val) +{ +pr_crit ("%s:%d: bad pud %016lx.\n" , file, line, val); +} +void __pgd_error (const char *file,int line,unsigned long val) {pr_crit ("%s:%d: bad pgd %016lx.\n" , file, line, val);
arch/arm64/mm/fault.c +1 -0
@@ -62,6 +62,7 @@ void show_pte(struct mm_struct *mm, unsigned long addr)break ; pud =pud_offset (pgd, addr); +printk (", *pud=%016llx" ,pud_val (*pud));if (pud_none (*pud) ||pud_bad (*pud))break ;
arch/arm64/mm/mmu.c +12 -4
@@ -32,6 +32,7 @@#include <asm/setup.h> #include <asm/sizes.h> #include <asm/tlb.h> +#include <asm/memblock.h> #include <asm/mmu_context.h> #include "mm.h"
@@ -190,9 +191,15 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,static void __init alloc_init_pud (pgd_t *pgd,unsigned long addr,unsigned long end,unsigned long phys) { -pud_t *pud =pud_offset (pgd, addr); +pud_t *pud;unsigned long next; +if (pgd_none (*pgd) ||pgd_bad (*pgd)) { + pud =early_alloc (PTRS_PER_PUD *sizeof (pud_t)); +pgd_populate (&init_mm, pgd, pud); + } + + pud =pud_offset (pgd, addr); do { next = pud_addr_end(addr, end);
@@ -262,10 +269,11 @@ static void __init map_mem(void)* memory addressable from the initial direct kernel mapping. * * The initial direct kernel mapping, located at swapper_pg_dir, -* gives us PGDIR_SIZE memory starting from PHYS_OFFSET (which must be -* aligned to 2MB as per Documentation/arm64/booting.txt). +* gives us PGDIR_SIZE (2 and 3 levels) or PUD_SIZE (4 levels) memory +* starting from PHYS_OFFSET (which must be aligned to 2MB as per +* Documentation/arm64/booting.txt). */ - limit =PHYS_OFFSET +PGDIR_SIZE ; + limit =PHYS_OFFSET +MEMBLOCK_INITIAL_LIMIT ;memblock_set_current_limit (limit);/* map all the memory banks */
Signature · 1 line
1.7.10.4