Skip to contentSkip to conversation

arm64: Add support for configuring the translation granule

PATCH In review Nikos Nikoleris 3 participants 5 messages Oct 29, 2020 – Oct 30, 2020 KVM

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 arm64: Add support for configuring the translation granule

Make the translation granule configurable for arm64. arm64 supports page sizes of 4K, 16K and 64K. By default, arm64 is configured with 64K pages. configure has been extended with a new argument:

--page-shift=(12|14|16)

which allows the user to set the page shift and therefore the page size for arm64. Using the --page-shift for any other architecture results an error message.

To allow for smaller page sizes and 42b VA, this change adds support for 4-level and 3-level page tables. At compile time, we determine how many levels in the page tables we needed.

---
 configure                     | 21 +++++++++++
 lib/arm/asm/page.h            |  4 ++
 lib/arm/asm/pgtable-hwdef.h   |  4 ++
 lib/arm/asm/pgtable.h         |  6 +++
 lib/arm/asm/thread_info.h     |  4 +-
 lib/arm64/asm/page.h          | 17 ++++++---
 lib/arm64/asm/pgtable-hwdef.h | 38 +++++++++++++------
 lib/arm64/asm/pgtable.h       | 69 +++++++++++++++++++++++++++++++++--
 lib/arm/mmu.c                 | 26 ++++++++-----
 arm/cstart64.S                | 12 +++++-
 10 files changed, 169 insertions(+), 32 deletions(-)
configure +21 -0
index 706aab5..94637ec 100755
@@ -25,6 +25,7 @@ vmm="qemu"
 errata_force=0
 erratatxt="$srcdir/errata.txt"
 host_key_document=
+page_shift=
 
 usage() {
     cat <<-EOF
@@ -105,6 +106,9 @@ while [[ "$1" = -* ]]; do
 	--host-key-document)
 	    host_key_document="$arg"
 	    ;;
+        --page-shift)
+            page_shift="$arg"
+            ;;
 	--help)
 	    usage
 	    ;;
@@ -123,6 +127,22 @@ arch_name=$arch
 [ "$arch" = "aarch64" ] && arch="arm64"
 [ "$arch_name" = "arm64" ] && arch_name="aarch64"
 
+if [ -z "$page_shift" ]; then
+    [ "$arch" = "arm64" ] && page_shift="16"
+    [ "$arch" = "arm" ] && page_shift="12"
+else
+    if [ "$arch" != "arm64" ]; then
+        echo "--page-shift is not supported for $arch"
+        usage
+    fi
+
+    if [ "$page_shift" != "12" ] && [ "$page_shift" != "14" ] &&
+           [ "$page_shift" != "16" ]; then
+        echo "Page shift of $page_shift not supported for arm64"
+        usage
+    fi
+fi
+
 [ -z "$processor" ] && processor="$arch"
 
 if [ "$processor" = "arm64" ]; then
@@ -254,6 +274,7 @@ cat <<EOF >> lib/config.h
 
 #define CONFIG_UART_EARLY_BASE ${arm_uart_early_addr}
 #define CONFIG_ERRATA_FORCE ${errata_force}
+#define CONFIG_PAGE_SHIFT ${page_shift}
 
 EOF
 fi
lib/arm/asm/page.h +4 -0
index 039c9f7..ae0ac2c 100644
@@ -29,6 +29,10 @@ typedef struct { pteval_t pgprot; } pgprot_t;
 #define pgd_val(x)		((x).pgd)
 #define pgprot_val(x)		((x).pgprot)
 
+/* For compatibility with arm64 page tables */
+#define pud_t pgd_t
+#define pud_val(x) pgd_val(x)
+
 #define __pte(x)		((pte_t) { (x) } )
 #define __pmd(x)		((pmd_t) { (x) } )
 #define __pgd(x)		((pgd_t) { (x) } )
lib/arm/asm/pgtable-hwdef.h +4 -0
index 4107e18..fe1d854 100644
@@ -19,6 +19,10 @@
 #define PTRS_PER_PTE		512
 #define PTRS_PER_PMD		512
 
+/* For compatibility with arm64 page tables */
+#define PUD_SIZE		PGDIR_SIZE
+#define PUD_MASK		PGDIR_MASK
+
 #define PMD_SHIFT		21
 #define PMD_SIZE		(_AC(1,UL) << PMD_SHIFT)
 #define PMD_MASK		(~((1 << PMD_SHIFT) - 1))
lib/arm/asm/pgtable.h +6 -0
index 078dd16..4759d82 100644
@@ -53,6 +53,12 @@ static inline pmd_t *pgd_page_vaddr(pgd_t pgd)
 	return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
 }
 
+/* For compatibility with arm64 page tables */
+#define pud_valid(pud)		pgd_valid(pud)
+#define pud_offset(pgd, addr)  ((pud_t *)pgd)
+#define pud_free(pud)
+#define pud_alloc(pgd, addr)   pud_offset(pgd, addr)
+
 #define pmd_index(addr) \
 	(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
 #define pmd_offset(pgd, addr) \
lib/arm/asm/thread_info.h +3 -1
index 80ab395..eaa7258 100644
@@ -14,10 +14,12 @@
 #define THREAD_SHIFT		PAGE_SHIFT
 #define THREAD_SIZE		PAGE_SIZE
 #define THREAD_MASK		PAGE_MASK
+#define THREAD_ALIGNMENT	PAGE_SIZE
 #else
 #define THREAD_SHIFT		MIN_THREAD_SHIFT
 #define THREAD_SIZE		(_AC(1,UL) << THREAD_SHIFT)
 #define THREAD_MASK		(~(THREAD_SIZE-1))
+#define THREAD_ALIGNMENT	THREAD_SIZE
 #endif
 
 #ifndef __ASSEMBLY__
@@ -38,7 +40,7 @@
 
 static inline void *thread_stack_alloc(void)
 {
-	void *sp = memalign(PAGE_SIZE, THREAD_SIZE);
+	void *sp = memalign(THREAD_ALIGNMENT, THREAD_SIZE);
 	return sp + THREAD_START_SP;
 }
 
lib/arm64/asm/page.h +11 -6
index 46af552..726a0c0 100644
@@ -10,38 +10,43 @@
  * This work is licensed under the terms of the GNU GPL, version 2.
  */
 
+#include <config.h>
 #include <linux/const.h>
 
-#define PGTABLE_LEVELS		2
 #define VA_BITS			42
 
-#define PAGE_SHIFT		16
+#define PAGE_SHIFT		CONFIG_PAGE_SHIFT
 #define PAGE_SIZE		(_AC(1,UL) << PAGE_SHIFT)
 #define PAGE_MASK		(~(PAGE_SIZE-1))
 
+#define PGTABLE_LEVELS		(((VA_BITS) - 4) / (PAGE_SHIFT - 3))
+
 #ifndef __ASSEMBLY__
 
 #define PAGE_ALIGN(addr)	ALIGN(addr, PAGE_SIZE)
 
 typedef u64 pteval_t;
 typedef u64 pmdval_t;
+typedef u64 pudval_t;
 typedef u64 pgdval_t;
 typedef struct { pteval_t pte; } pte_t;
+typedef struct { pmdval_t pmd; } pmd_t;
+typedef struct { pudval_t pud; } pud_t;
 typedef struct { pgdval_t pgd; } pgd_t;
 typedef struct { pteval_t pgprot; } pgprot_t;
 
 #define pte_val(x)		((x).pte)
+#define pmd_val(x)		((x).pmd)
+#define pud_val(x)		((x).pud)
 #define pgd_val(x)		((x).pgd)
 #define pgprot_val(x)		((x).pgprot)
 
 #define __pte(x)		((pte_t) { (x) } )
+#define __pmd(x)		((pmd_t) { (x) } )
+#define __pud(x)		((pud_t) { (x) } )
 #define __pgd(x)		((pgd_t) { (x) } )
 #define __pgprot(x)		((pgprot_t) { (x) } )
 
-typedef struct { pgd_t pgd; } pmd_t;
-#define pmd_val(x)		(pgd_val((x).pgd))
-#define __pmd(x)		((pmd_t) { __pgd(x) } )
-
 #define __va(x)			((void *)__phys_to_virt((phys_addr_t)(x)))
 #define __pa(x)			__virt_to_phys((unsigned long)(x))
 
lib/arm64/asm/pgtable-hwdef.h +27 -11
index 3352489..f9110e1 100644
@@ -9,38 +9,54 @@
  * This work is licensed under the terms of the GNU GPL, version 2.
  */
 
+#include <asm/page.h>
+
 #define UL(x) _AC(x, UL)
 
+#define PGTABLE_LEVEL_SHIFT(n)	((PAGE_SHIFT - 3) * (4 - (n)) + 3)
 #define PTRS_PER_PTE		(1 << (PAGE_SHIFT - 3))
 
+#if PGTABLE_LEVELS > 2
+#define PMD_SHIFT		PGTABLE_LEVEL_SHIFT(2)
+#define PTRS_PER_PMD		PTRS_PER_PTE
+#define PMD_SIZE		(UL(1) << PMD_SHIFT)
+#define PMD_MASK		(~(PMD_SIZE-1))
+#endif
+
+#if PGTABLE_LEVELS > 3
+#define PUD_SHIFT		PGTABLE_LEVEL_SHIFT(1)
+#define PTRS_PER_PUD		PTRS_PER_PTE
+#define PUD_SIZE		(UL(1) << PUD_SHIFT)
+#define PUD_MASK		(~(PUD_SIZE-1))
+#else
+#define PUD_SIZE                PGDIR_SIZE
+#define PUD_MASK                PGDIR_MASK
+#endif
+
+#define PUD_VALID		(_AT(pudval_t, 1) << 0)
+
 /*
  * PGDIR_SHIFT determines the size a top-level page table entry can map
  * (depending on the configuration, this level can be 0, 1 or 2).
  */
-#define PGDIR_SHIFT		((PAGE_SHIFT - 3) * PGTABLE_LEVELS + 3)
+#define PGDIR_SHIFT		PGTABLE_LEVEL_SHIFT(4 - PGTABLE_LEVELS)
 #define PGDIR_SIZE		(_AC(1, UL) << PGDIR_SHIFT)
 #define PGDIR_MASK		(~(PGDIR_SIZE-1))
 #define PTRS_PER_PGD		(1 << (VA_BITS - PGDIR_SHIFT))
 
 #define PGD_VALID		(_AT(pgdval_t, 1) << 0)
 
-/* From include/asm-generic/pgtable-nopmd.h */
-#define PMD_SHIFT		PGDIR_SHIFT
-#define PTRS_PER_PMD		1
-#define PMD_SIZE		(UL(1) << PMD_SHIFT)
-#define PMD_MASK		(~(PMD_SIZE-1))
-
 /*
  * Section address mask and size definitions.
  */
-#define SECTION_SHIFT		PMD_SHIFT
-#define SECTION_SIZE		(_AC(1, UL) << SECTION_SHIFT)
-#define SECTION_MASK		(~(SECTION_SIZE-1))
+#define SECTION_SHIFT          PMD_SHIFT
+#define SECTION_SIZE           (_AC(1, UL) << SECTION_SHIFT)
+#define SECTION_MASK           (~(SECTION_SIZE-1))
 
 /*
  * Hardware page table definitions.
  *
- * Level 1 descriptor (PMD).
+ * Level 0,1,2 descriptor (PGG, PUD and PMD).
  */
 #define PMD_TYPE_MASK		(_AT(pmdval_t, 3) << 0)
 #define PMD_TYPE_FAULT		(_AT(pmdval_t, 0) << 0)
lib/arm64/asm/pgtable.h +66 -3

91 lines not on this page · raw message

lib/arm/mmu.c +16 -10
index 540a1e8..6d1c75b 100644
@@ -81,7 +81,8 @@ void mmu_disable(void)
 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
 {
 	pgd_t *pgd = pgd_offset(pgtable, vaddr);
-	pmd_t *pmd = pmd_alloc(pgd, vaddr);
+	pud_t *pud = pud_alloc(pgd, vaddr);
+	pmd_t *pmd = pmd_alloc(pud, vaddr);
 	pte_t *pte = pte_alloc(pmd, vaddr);
 
 	return &pte_val(*pte);
@@ -133,18 +134,20 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset,
 			phys_addr_t phys_start, phys_addr_t phys_end,
 			pgprot_t prot)
 {
-	phys_addr_t paddr = phys_start & PGDIR_MASK;
-	uintptr_t vaddr = virt_offset & PGDIR_MASK;
+	phys_addr_t paddr = phys_start & PUD_MASK;
+	uintptr_t vaddr = virt_offset & PUD_MASK;
 	uintptr_t virt_end = phys_end - paddr + vaddr;
 	pgd_t *pgd;
-	pgd_t entry;
+	pud_t *pud;
+	pud_t entry;
 
-	for (; vaddr < virt_end; vaddr += PGDIR_SIZE, paddr += PGDIR_SIZE) {
-		pgd_val(entry) = paddr;
-		pgd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
-		pgd_val(entry) |= pgprot_val(prot);
+	for (; vaddr < virt_end; vaddr += PUD_SIZE, paddr += PUD_SIZE) {
+		pud_val(entry) = paddr;
+		pud_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
+		pud_val(entry) |= pgprot_val(prot);
 		pgd = pgd_offset(pgtable, vaddr);
-		WRITE_ONCE(*pgd, entry);
+		pud = pud_alloc(pgd, vaddr);
+		WRITE_ONCE(*pud, entry);
 		flush_tlb_page(vaddr);
 	}
 }
@@ -207,6 +210,7 @@ unsigned long __phys_to_virt(phys_addr_t addr)
 void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
 {
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 
@@ -215,7 +219,9 @@ void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
 
 	pgd = pgd_offset(pgtable, vaddr);
 	assert(pgd_valid(*pgd));
-	pmd = pmd_offset(pgd, vaddr);
+	pud = pud_offset(pgd, vaddr);
+	assert(pud_valid(*pud));
+	pmd = pmd_offset(pud, vaddr);
 	assert(pmd_valid(*pmd));
 
 	if (pmd_huge(*pmd)) {
arm/cstart64.S +11 -1
index ffdd49f..530ffb6 100644
@@ -157,6 +157,16 @@ halt:
  */
 #define MAIR(attr, mt) ((attr) << ((mt) * 8))
 
+#if PAGE_SHIFT == 16
+#define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K
+#elif PAGE_SHIFT == 14
+#define TCR_TG_FLAGS	TCR_TG0_16K | TCR_TG1_16K
+#elif PAGE_SHIFT == 12
+#define TCR_TG_FLAGS	TCR_TG0_4K | TCR_TG1_4K
+#else
+#error Unsupported PAGE_SHIFT
+#endif
+
 .globl asm_mmu_enable
 asm_mmu_enable:
 	tlbi	vmalle1			// invalidate I + D TLBs
@@ -164,7 +174,7 @@ asm_mmu_enable:
 
 	/* TCR */
 	ldr	x1, =TCR_TxSZ(VA_BITS) |		\
-		     TCR_TG0_64K | TCR_TG1_64K |	\
+		     TCR_TG_FLAGS  |			\
 		     TCR_IRGN_WBWA | TCR_ORGN_WBWA |	\
 		     TCR_SHARED
 	mrs	x2, id_aa64mmfr0_el1
Signature · 1 line
2.17.1
#1 lore raw
Quoted 10 lines · Nikos Nikoleris

Make the translation granule configurable for arm64. arm64 supports page sizes of 4K, 16K and 64K. By default, arm64 is configured with 64K pages. configure has been extended with a new argument:

--page-shift=(12|14|16)

which allows the user to set the page shift and therefore the page size for arm64. Using the --page-shift for any other architecture results an error message.

Quoted 5 lines

To allow for smaller page sizes and 42b VA, this change adds support for 4-level and 3-level page tables. At compile time, we determine how many levels in the page tables we needed.

Thanks for this Nikos! It looks good to me and I'll give it a test drive soon.

drew

#2 lore raw
In reply to Nikos Nikoleris

Hi Nikos,

I scanned the patches, will try to do a proper review next week. Some suggestions below.

Quoted 5 lines · Nikos Nikoleris

Make the translation granule configurable for arm64. arm64 supports page sizes of 4K, 16K and 64K. By default, arm64 is configured with 64K pages. configure has been extended with a new argument:

--page-shift=(12|14|16)

How about --page-size=4K/16K/64K, which accepts lower and upper case 'k'?

It might be just me, because I'm not familiar with memory management, but each time I see page-shift I need to do the math in my head to get to the page size. I think page-size is more intuitive. It's also somewhat similar to the kernel config (CONFIG_ARM64_4K_PAGES, for example, and not CONFIG_ARM64_PAGE_SHIFT_12).

I might have missed it in the patch, how about printing an error message when the configured page size is not supported by the hardware? kvm-unit-tests runs C code before creating the page tables, and the UART is available very early, so it shouldn't be too hard. We can put the check in setup_mmu().

Thanks,
Alex

Quoted 462 lines

which allows the user to set the page shift and therefore the page size for arm64. Using the --page-shift for any other architecture results an error message.

To allow for smaller page sizes and 42b VA, this change adds support for 4-level and 3-level page tables. At compile time, we determine how many levels in the page tables we needed.

---
 configure                     | 21 +++++++++++
 lib/arm/asm/page.h            |  4 ++
 lib/arm/asm/pgtable-hwdef.h   |  4 ++
 lib/arm/asm/pgtable.h         |  6 +++
 lib/arm/asm/thread_info.h     |  4 +-
 lib/arm64/asm/page.h          | 17 ++++++---
 lib/arm64/asm/pgtable-hwdef.h | 38 +++++++++++++------
 lib/arm64/asm/pgtable.h       | 69 +++++++++++++++++++++++++++++++++--
 lib/arm/mmu.c                 | 26 ++++++++-----
 arm/cstart64.S                | 12 +++++-
 10 files changed, 169 insertions(+), 32 deletions(-)
configure +21 -0
index 706aab5..94637ec 100755
@@ -25,6 +25,7 @@ vmm="qemu"
 errata_force=0
 erratatxt="$srcdir/errata.txt"
 host_key_document=
+page_shift=
 
 usage() {
     cat <<-EOF
@@ -105,6 +106,9 @@ while [[ "$1" = -* ]]; do
 	--host-key-document)
 	    host_key_document="$arg"
 	    ;;
+        --page-shift)
+            page_shift="$arg"
+            ;;
 	--help)
 	    usage
 	    ;;
@@ -123,6 +127,22 @@ arch_name=$arch
 [ "$arch" = "aarch64" ] && arch="arm64"
 [ "$arch_name" = "arm64" ] && arch_name="aarch64"
 
+if [ -z "$page_shift" ]; then
+    [ "$arch" = "arm64" ] && page_shift="16"
+    [ "$arch" = "arm" ] && page_shift="12"
+else
+    if [ "$arch" != "arm64" ]; then
+        echo "--page-shift is not supported for $arch"
+        usage
+    fi
+
+    if [ "$page_shift" != "12" ] && [ "$page_shift" != "14" ] &&
+           [ "$page_shift" != "16" ]; then
+        echo "Page shift of $page_shift not supported for arm64"
+        usage
+    fi
+fi
+
 [ -z "$processor" ] && processor="$arch"
 
 if [ "$processor" = "arm64" ]; then
@@ -254,6 +274,7 @@ cat <<EOF >> lib/config.h
 
 #define CONFIG_UART_EARLY_BASE ${arm_uart_early_addr}
 #define CONFIG_ERRATA_FORCE ${errata_force}
+#define CONFIG_PAGE_SHIFT ${page_shift}
 
 EOF
 fi
lib/arm/asm/page.h +4 -0
index 039c9f7..ae0ac2c 100644
@@ -29,6 +29,10 @@ typedef struct { pteval_t pgprot; } pgprot_t;
 #define pgd_val(x)		((x).pgd)
 #define pgprot_val(x)		((x).pgprot)
 
+/* For compatibility with arm64 page tables */
+#define pud_t pgd_t
+#define pud_val(x) pgd_val(x)
+
 #define __pte(x)		((pte_t) { (x) } )
 #define __pmd(x)		((pmd_t) { (x) } )
 #define __pgd(x)		((pgd_t) { (x) } )
lib/arm/asm/pgtable-hwdef.h +4 -0
index 4107e18..fe1d854 100644
@@ -19,6 +19,10 @@
 #define PTRS_PER_PTE		512
 #define PTRS_PER_PMD		512
 
+/* For compatibility with arm64 page tables */
+#define PUD_SIZE		PGDIR_SIZE
+#define PUD_MASK		PGDIR_MASK
+
 #define PMD_SHIFT		21
 #define PMD_SIZE		(_AC(1,UL) << PMD_SHIFT)
 #define PMD_MASK		(~((1 << PMD_SHIFT) - 1))
lib/arm/asm/pgtable.h +6 -0
index 078dd16..4759d82 100644
@@ -53,6 +53,12 @@ static inline pmd_t *pgd_page_vaddr(pgd_t pgd)
 	return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
 }
 
+/* For compatibility with arm64 page tables */
+#define pud_valid(pud)		pgd_valid(pud)
+#define pud_offset(pgd, addr)  ((pud_t *)pgd)
+#define pud_free(pud)
+#define pud_alloc(pgd, addr)   pud_offset(pgd, addr)
+
 #define pmd_index(addr) \
 	(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
 #define pmd_offset(pgd, addr) \
lib/arm/asm/thread_info.h +3 -1
index 80ab395..eaa7258 100644
@@ -14,10 +14,12 @@
 #define THREAD_SHIFT		PAGE_SHIFT
 #define THREAD_SIZE		PAGE_SIZE
 #define THREAD_MASK		PAGE_MASK
+#define THREAD_ALIGNMENT	PAGE_SIZE
 #else
 #define THREAD_SHIFT		MIN_THREAD_SHIFT
 #define THREAD_SIZE		(_AC(1,UL) << THREAD_SHIFT)
 #define THREAD_MASK		(~(THREAD_SIZE-1))
+#define THREAD_ALIGNMENT	THREAD_SIZE
 #endif
 
 #ifndef __ASSEMBLY__
@@ -38,7 +40,7 @@
 
 static inline void *thread_stack_alloc(void)
 {
-	void *sp = memalign(PAGE_SIZE, THREAD_SIZE);
+	void *sp = memalign(THREAD_ALIGNMENT, THREAD_SIZE);
 	return sp + THREAD_START_SP;
 }
 
lib/arm64/asm/page.h +11 -6
index 46af552..726a0c0 100644
@@ -10,38 +10,43 @@
  * This work is licensed under the terms of the GNU GPL, version 2.
  */
 
+#include <config.h>
 #include <linux/const.h>
 
-#define PGTABLE_LEVELS		2
 #define VA_BITS			42
 
-#define PAGE_SHIFT		16
+#define PAGE_SHIFT		CONFIG_PAGE_SHIFT
 #define PAGE_SIZE		(_AC(1,UL) << PAGE_SHIFT)
 #define PAGE_MASK		(~(PAGE_SIZE-1))
 
+#define PGTABLE_LEVELS		(((VA_BITS) - 4) / (PAGE_SHIFT - 3))
+
 #ifndef __ASSEMBLY__
 
 #define PAGE_ALIGN(addr)	ALIGN(addr, PAGE_SIZE)
 
 typedef u64 pteval_t;
 typedef u64 pmdval_t;
+typedef u64 pudval_t;
 typedef u64 pgdval_t;
 typedef struct { pteval_t pte; } pte_t;
+typedef struct { pmdval_t pmd; } pmd_t;
+typedef struct { pudval_t pud; } pud_t;
 typedef struct { pgdval_t pgd; } pgd_t;
 typedef struct { pteval_t pgprot; } pgprot_t;
 
 #define pte_val(x)		((x).pte)
+#define pmd_val(x)		((x).pmd)
+#define pud_val(x)		((x).pud)
 #define pgd_val(x)		((x).pgd)
 #define pgprot_val(x)		((x).pgprot)
 
 #define __pte(x)		((pte_t) { (x) } )
+#define __pmd(x)		((pmd_t) { (x) } )
+#define __pud(x)		((pud_t) { (x) } )
 #define __pgd(x)		((pgd_t) { (x) } )
 #define __pgprot(x)		((pgprot_t) { (x) } )
 
-typedef struct { pgd_t pgd; } pmd_t;
-#define pmd_val(x)		(pgd_val((x).pgd))
-#define __pmd(x)		((pmd_t) { __pgd(x) } )
-
 #define __va(x)			((void *)__phys_to_virt((phys_addr_t)(x)))
 #define __pa(x)			__virt_to_phys((unsigned long)(x))
 
lib/arm64/asm/pgtable-hwdef.h +27 -11
index 3352489..f9110e1 100644
@@ -9,38 +9,54 @@
  * This work is licensed under the terms of the GNU GPL, version 2.
  */
 
+#include <asm/page.h>
+
 #define UL(x) _AC(x, UL)
 
+#define PGTABLE_LEVEL_SHIFT(n)	((PAGE_SHIFT - 3) * (4 - (n)) + 3)
 #define PTRS_PER_PTE		(1 << (PAGE_SHIFT - 3))
 
+#if PGTABLE_LEVELS > 2
+#define PMD_SHIFT		PGTABLE_LEVEL_SHIFT(2)
+#define PTRS_PER_PMD		PTRS_PER_PTE
+#define PMD_SIZE		(UL(1) << PMD_SHIFT)
+#define PMD_MASK		(~(PMD_SIZE-1))
+#endif
+
+#if PGTABLE_LEVELS > 3
+#define PUD_SHIFT		PGTABLE_LEVEL_SHIFT(1)
+#define PTRS_PER_PUD		PTRS_PER_PTE
+#define PUD_SIZE		(UL(1) << PUD_SHIFT)
+#define PUD_MASK		(~(PUD_SIZE-1))
+#else
+#define PUD_SIZE                PGDIR_SIZE
+#define PUD_MASK                PGDIR_MASK
+#endif
+
+#define PUD_VALID		(_AT(pudval_t, 1) << 0)
+
 /*
  * PGDIR_SHIFT determines the size a top-level page table entry can map
  * (depending on the configuration, this level can be 0, 1 or 2).
  */
-#define PGDIR_SHIFT		((PAGE_SHIFT - 3) * PGTABLE_LEVELS + 3)
+#define PGDIR_SHIFT		PGTABLE_LEVEL_SHIFT(4 - PGTABLE_LEVELS)
 #define PGDIR_SIZE		(_AC(1, UL) << PGDIR_SHIFT)
 #define PGDIR_MASK		(~(PGDIR_SIZE-1))
 #define PTRS_PER_PGD		(1 << (VA_BITS - PGDIR_SHIFT))
 
 #define PGD_VALID		(_AT(pgdval_t, 1) << 0)
 
-/* From include/asm-generic/pgtable-nopmd.h */
-#define PMD_SHIFT		PGDIR_SHIFT
-#define PTRS_PER_PMD		1
-#define PMD_SIZE		(UL(1) << PMD_SHIFT)
-#define PMD_MASK		(~(PMD_SIZE-1))
-
 /*
  * Section address mask and size definitions.
  */
-#define SECTION_SHIFT		PMD_SHIFT
-#define SECTION_SIZE		(_AC(1, UL) << SECTION_SHIFT)
-#define SECTION_MASK		(~(SECTION_SIZE-1))
+#define SECTION_SHIFT          PMD_SHIFT
+#define SECTION_SIZE           (_AC(1, UL) << SECTION_SHIFT)
+#define SECTION_MASK           (~(SECTION_SIZE-1))
 
 /*
  * Hardware page table definitions.
  *
- * Level 1 descriptor (PMD).
+ * Level 0,1,2 descriptor (PGG, PUD and PMD).
  */
 #define PMD_TYPE_MASK		(_AT(pmdval_t, 3) << 0)
 #define PMD_TYPE_FAULT		(_AT(pmdval_t, 0) << 0)
lib/arm64/asm/pgtable.h +66 -3
index e577d9c..c7632ae 100644
@@ -30,10 +30,12 @@
 #define pgtable_pa(x)		((unsigned long)(x))
 
 #define pgd_none(pgd)		(!pgd_val(pgd))
+#define pud_none(pud)		(!pud_val(pud))
 #define pmd_none(pmd)		(!pmd_val(pmd))
 #define pte_none(pte)		(!pte_val(pte))
 
 #define pgd_valid(pgd)		(pgd_val(pgd) & PGD_VALID)
+#define pud_valid(pud)		(pud_val(pud) & PUD_VALID)
 #define pmd_valid(pmd)		(pmd_val(pmd) & PMD_SECT_VALID)
 #define pte_valid(pte)		(pte_val(pte) & PTE_VALID)
 
@@ -52,15 +54,76 @@ static inline pgd_t *pgd_alloc(void)
 	return pgd;
 }
 
-#define pmd_offset(pgd, addr)	((pmd_t *)pgd)
-#define pmd_free(pmd)
-#define pmd_alloc(pgd, addr)	pmd_offset(pgd, addr)
+static inline pud_t *pgd_page_vaddr(pgd_t pgd)
+{
+	return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
+}
+
+static inline pmd_t *pud_page_vaddr(pud_t pud)
+{
+	return pgtable_va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
+}
+
 
 static inline pte_t *pmd_page_vaddr(pmd_t pmd)
 {
 	return pgtable_va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK);
 }
 
+#if PGTABLE_LEVELS > 2
+#define pmd_index(addr)					\
+	(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
+#define pmd_offset(pud, addr)				\
+	(pud_page_vaddr(*(pud)) + pmd_index(addr))
+#define pmd_free(pmd) free_page(pmd)
+static inline pmd_t *pmd_alloc_one(void)
+{
+	assert(PTRS_PER_PMD * sizeof(pmd_t) == PAGE_SIZE);
+	pmd_t *pmd = alloc_page();
+	return pmd;
+}
+static inline pmd_t *pmd_alloc(pud_t *pud, unsigned long addr)
+{
+        if (pud_none(*pud)) {
+		pud_t entry;
+		pud_val(entry) = pgtable_pa(pmd_alloc_one()) | PMD_TYPE_TABLE;
+		WRITE_ONCE(*pud, entry);
+	}
+	return pmd_offset(pud, addr);
+}
+#else
+#define pmd_offset(pud, addr)  ((pmd_t *)pud)
+#define pmd_free(pmd)
+#define pmd_alloc(pud, addr)   pmd_offset(pud, addr)
+#endif
+
+#if PGTABLE_LEVELS > 3
+#define pud_index(addr)                                 \
+	(((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
+#define pud_offset(pgd, addr)                           \
+	(pgd_page_vaddr(*(pgd)) + pud_index(addr))
+#define pud_free(pud) free_page(pud)
+static inline pud_t *pud_alloc_one(void)
+{
+	assert(PTRS_PER_PMD * sizeof(pud_t) == PAGE_SIZE);
+	pud_t *pud = alloc_page();
+	return pud;
+}
+static inline pud_t *pud_alloc(pgd_t *pgd, unsigned long addr)
+{
+	if (pgd_none(*pgd)) {
+		pgd_t entry;
+		pgd_val(entry) = pgtable_pa(pud_alloc_one()) | PMD_TYPE_TABLE;
+		WRITE_ONCE(*pgd, entry);
+	}
+	return pud_offset(pgd, addr);
+}
+#else
+#define pud_offset(pgd, addr)  ((pud_t *)pgd)
+#define pud_free(pud)
+#define pud_alloc(pgd, addr)   pud_offset(pgd, addr)
+#endif
+
 #define pte_index(addr) \
 	(((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
 #define pte_offset(pmd, addr) \
lib/arm/mmu.c +16 -10
index 540a1e8..6d1c75b 100644
@@ -81,7 +81,8 @@ void mmu_disable(void)
 static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
 {
 	pgd_t *pgd = pgd_offset(pgtable, vaddr);
-	pmd_t *pmd = pmd_alloc(pgd, vaddr);
+	pud_t *pud = pud_alloc(pgd, vaddr);
+	pmd_t *pmd = pmd_alloc(pud, vaddr);
 	pte_t *pte = pte_alloc(pmd, vaddr);
 
 	return &pte_val(*pte);
@@ -133,18 +134,20 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset,
 			phys_addr_t phys_start, phys_addr_t phys_end,
 			pgprot_t prot)
 {
-	phys_addr_t paddr = phys_start & PGDIR_MASK;
-	uintptr_t vaddr = virt_offset & PGDIR_MASK;
+	phys_addr_t paddr = phys_start & PUD_MASK;
+	uintptr_t vaddr = virt_offset & PUD_MASK;
 	uintptr_t virt_end = phys_end - paddr + vaddr;
 	pgd_t *pgd;
-	pgd_t entry;
+	pud_t *pud;
+	pud_t entry;
 
-	for (; vaddr < virt_end; vaddr += PGDIR_SIZE, paddr += PGDIR_SIZE) {
-		pgd_val(entry) = paddr;
-		pgd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
-		pgd_val(entry) |= pgprot_val(prot);
+	for (; vaddr < virt_end; vaddr += PUD_SIZE, paddr += PUD_SIZE) {
+		pud_val(entry) = paddr;
+		pud_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
+		pud_val(entry) |= pgprot_val(prot);
 		pgd = pgd_offset(pgtable, vaddr);
-		WRITE_ONCE(*pgd, entry);
+		pud = pud_alloc(pgd, vaddr);
+		WRITE_ONCE(*pud, entry);
 		flush_tlb_page(vaddr);
 	}
 }
@@ -207,6 +210,7 @@ unsigned long __phys_to_virt(phys_addr_t addr)
 void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
 {
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 
@@ -215,7 +219,9 @@ void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
 
 	pgd = pgd_offset(pgtable, vaddr);
 	assert(pgd_valid(*pgd));
-	pmd = pmd_offset(pgd, vaddr);
+	pud = pud_offset(pgd, vaddr);
+	assert(pud_valid(*pud));
+	pmd = pmd_offset(pud, vaddr);
 	assert(pmd_valid(*pmd));
 
 	if (pmd_huge(*pmd)) {
arm/cstart64.S +11 -1
index ffdd49f..530ffb6 100644
@@ -157,6 +157,16 @@ halt:
  */
 #define MAIR(attr, mt) ((attr) << ((mt) * 8))
 
+#if PAGE_SHIFT == 16
+#define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K
+#elif PAGE_SHIFT == 14
+#define TCR_TG_FLAGS	TCR_TG0_16K | TCR_TG1_16K
+#elif PAGE_SHIFT == 12
+#define TCR_TG_FLAGS	TCR_TG0_4K | TCR_TG1_4K
+#else
+#error Unsupported PAGE_SHIFT
+#endif
+
 .globl asm_mmu_enable
 asm_mmu_enable:
 	tlbi	vmalle1			// invalidate I + D TLBs
@@ -164,7 +174,7 @@ asm_mmu_enable:
 
 	/* TCR */
 	ldr	x1, =TCR_TxSZ(VA_BITS) |		\
-		     TCR_TG0_64K | TCR_TG1_64K |	\
+		     TCR_TG_FLAGS  |			\
 		     TCR_IRGN_WBWA | TCR_ORGN_WBWA |	\
 		     TCR_SHARED
 	mrs	x2, id_aa64mmfr0_el1
#3 lore raw

Hi Alex,

Quoted 5 lines · Alexandru Elisei

Hi Nikos,

I scanned the patches, will try to do a proper review next week. Some suggestions below.

Thanks for the review!

Quoted 9 lines · Nikos Nikoleris
Quoted 5 lines

Make the translation granule configurable for arm64. arm64 supports page sizes of 4K, 16K and 64K. By default, arm64 is configured with 64K pages. configure has been extended with a new argument:

--page-shift=(12|14|16)

How about --page-size=4K/16K/64K, which accepts lower and upper case 'k'?

Quoted 1 line

It might be just me, because I'm not familiar with memory management,

but each

Quoted 4 lines

time I see page-shift I need to do the math in my head to get to the page size. I think page-size is more intuitive. It's also somewhat similar to the kernel config (CONFIG_ARM64_4K_PAGES, for example, and not CONFIG_ARM64_PAGE_SHIFT_12).

You're right setting the page size is more intiutive. I had started implementing it that way but I got worried about the added complexity. I tried it again now and it doesn't look that bad. I'll post a v2 that allows you to configure the page size instead.

Quoted 5 lines

I might have missed it in the patch, how about printing an error message when the configured page size is not supported by the hardware? kvm-unit-tests runs C code before creating the page tables, and the UART is available very early, so it shouldn't be too hard. We can put the check in setup_mmu().

It's not there and it makes sense to check. Right now if we configure an unsupported page size, tests appear to be running just fine - you only find out when they time out.

Thanks,

Nikos

Quoted 464 lines

Thanks,
Alex

Quoted 462 lines

which allows the user to set the page shift and therefore the page size for arm64. Using the --page-shift for any other architecture results an error message.

To allow for smaller page sizes and 42b VA, this change adds support for 4-level and 3-level page tables. At compile time, we determine how many levels in the page tables we needed.

---
  configure                     | 21 +++++++++++
  lib/arm/asm/page.h            |  4 ++
  lib/arm/asm/pgtable-hwdef.h   |  4 ++
  lib/arm/asm/pgtable.h         |  6 +++
  lib/arm/asm/thread_info.h     |  4 +-
  lib/arm64/asm/page.h          | 17 ++++++---
  lib/arm64/asm/pgtable-hwdef.h | 38 +++++++++++++------
  lib/arm64/asm/pgtable.h       | 69 +++++++++++++++++++++++++++++++++--
  lib/arm/mmu.c                 | 26 ++++++++-----
  arm/cstart64.S                | 12 +++++-
  10 files changed, 169 insertions(+), 32 deletions(-)
configure +21 -0
index 706aab5..94637ec 100755
@@ -25,6 +25,7 @@ vmm="qemu"
  errata_force=0
  erratatxt="$srcdir/errata.txt"
  host_key_document=
+page_shift=
 
  usage() {
      cat <<-EOF
@@ -105,6 +106,9 @@ while [[ "$1" = -* ]]; do
  	--host-key-document)
  	    host_key_document="$arg"
  	    ;;
+        --page-shift)
+            page_shift="$arg"
+            ;;
  	--help)
  	    usage
  	    ;;
@@ -123,6 +127,22 @@ arch_name=$arch
  [ "$arch" = "aarch64" ] && arch="arm64"
  [ "$arch_name" = "arm64" ] && arch_name="aarch64"
 
+if [ -z "$page_shift" ]; then
+    [ "$arch" = "arm64" ] && page_shift="16"
+    [ "$arch" = "arm" ] && page_shift="12"
+else
+    if [ "$arch" != "arm64" ]; then
+        echo "--page-shift is not supported for $arch"
+        usage
+    fi
+
+    if [ "$page_shift" != "12" ] && [ "$page_shift" != "14" ] &&
+           [ "$page_shift" != "16" ]; then
+        echo "Page shift of $page_shift not supported for arm64"
+        usage
+    fi
+fi
+
  [ -z "$processor" ] && processor="$arch"
 
  if [ "$processor" = "arm64" ]; then
@@ -254,6 +274,7 @@ cat <<EOF >> lib/config.h
 
  #define CONFIG_UART_EARLY_BASE ${arm_uart_early_addr}
  #define CONFIG_ERRATA_FORCE ${errata_force}
+#define CONFIG_PAGE_SHIFT ${page_shift}
 
  EOF
  fi
lib/arm/asm/page.h +4 -0
index 039c9f7..ae0ac2c 100644
@@ -29,6 +29,10 @@ typedef struct { pteval_t pgprot; } pgprot_t;
  #define pgd_val(x)		((x).pgd)
  #define pgprot_val(x)		((x).pgprot)
 
+/* For compatibility with arm64 page tables */
+#define pud_t pgd_t
+#define pud_val(x) pgd_val(x)
+
  #define __pte(x)		((pte_t) { (x) } )
  #define __pmd(x)		((pmd_t) { (x) } )
  #define __pgd(x)		((pgd_t) { (x) } )
lib/arm/asm/pgtable-hwdef.h +4 -0
index 4107e18..fe1d854 100644
@@ -19,6 +19,10 @@
  #define PTRS_PER_PTE		512
  #define PTRS_PER_PMD		512
 
+/* For compatibility with arm64 page tables */
+#define PUD_SIZE		PGDIR_SIZE
+#define PUD_MASK		PGDIR_MASK
+
  #define PMD_SHIFT		21
  #define PMD_SIZE		(_AC(1,UL) << PMD_SHIFT)
  #define PMD_MASK		(~((1 << PMD_SHIFT) - 1))
lib/arm/asm/pgtable.h +6 -0
index 078dd16..4759d82 100644
@@ -53,6 +53,12 @@ static inline pmd_t *pgd_page_vaddr(pgd_t pgd)
  	return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
  }
 
+/* For compatibility with arm64 page tables */
+#define pud_valid(pud)		pgd_valid(pud)
+#define pud_offset(pgd, addr)  ((pud_t *)pgd)
+#define pud_free(pud)
+#define pud_alloc(pgd, addr)   pud_offset(pgd, addr)
+
  #define pmd_index(addr) \
  	(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
  #define pmd_offset(pgd, addr) \
lib/arm/asm/thread_info.h +3 -1
index 80ab395..eaa7258 100644
@@ -14,10 +14,12 @@
  #define THREAD_SHIFT		PAGE_SHIFT
  #define THREAD_SIZE		PAGE_SIZE
  #define THREAD_MASK		PAGE_MASK
+#define THREAD_ALIGNMENT	PAGE_SIZE
  #else
  #define THREAD_SHIFT		MIN_THREAD_SHIFT
  #define THREAD_SIZE		(_AC(1,UL) << THREAD_SHIFT)
  #define THREAD_MASK		(~(THREAD_SIZE-1))
+#define THREAD_ALIGNMENT	THREAD_SIZE
  #endif
 
  #ifndef __ASSEMBLY__
@@ -38,7 +40,7 @@
 
  static inline void *thread_stack_alloc(void)
  {
-	void *sp = memalign(PAGE_SIZE, THREAD_SIZE);
+	void *sp = memalign(THREAD_ALIGNMENT, THREAD_SIZE);
  	return sp + THREAD_START_SP;
  }
 
lib/arm64/asm/page.h +11 -6
index 46af552..726a0c0 100644
@@ -10,38 +10,43 @@
   * This work is licensed under the terms of the GNU GPL, version 2.
   */
 
+#include <config.h>
  #include <linux/const.h>
 
-#define PGTABLE_LEVELS		2
  #define VA_BITS			42
 
-#define PAGE_SHIFT		16
+#define PAGE_SHIFT		CONFIG_PAGE_SHIFT
  #define PAGE_SIZE		(_AC(1,UL) << PAGE_SHIFT)
  #define PAGE_MASK		(~(PAGE_SIZE-1))
 
+#define PGTABLE_LEVELS		(((VA_BITS) - 4) / (PAGE_SHIFT - 3))
+
  #ifndef __ASSEMBLY__
 
  #define PAGE_ALIGN(addr)	ALIGN(addr, PAGE_SIZE)
 
  typedef u64 pteval_t;
  typedef u64 pmdval_t;
+typedef u64 pudval_t;
  typedef u64 pgdval_t;
  typedef struct { pteval_t pte; } pte_t;
+typedef struct { pmdval_t pmd; } pmd_t;
+typedef struct { pudval_t pud; } pud_t;
  typedef struct { pgdval_t pgd; } pgd_t;
  typedef struct { pteval_t pgprot; } pgprot_t;
 
  #define pte_val(x)		((x).pte)
+#define pmd_val(x)		((x).pmd)
+#define pud_val(x)		((x).pud)
  #define pgd_val(x)		((x).pgd)
  #define pgprot_val(x)		((x).pgprot)
 
  #define __pte(x)		((pte_t) { (x) } )
+#define __pmd(x)		((pmd_t) { (x) } )
+#define __pud(x)		((pud_t) { (x) } )
  #define __pgd(x)		((pgd_t) { (x) } )
  #define __pgprot(x)		((pgprot_t) { (x) } )
 
-typedef struct { pgd_t pgd; } pmd_t;
-#define pmd_val(x)		(pgd_val((x).pgd))
-#define __pmd(x)		((pmd_t) { __pgd(x) } )
-
  #define __va(x)			((void *)__phys_to_virt((phys_addr_t)(x)))
  #define __pa(x)			__virt_to_phys((unsigned long)(x))
 
lib/arm64/asm/pgtable-hwdef.h +27 -11
index 3352489..f9110e1 100644
@@ -9,38 +9,54 @@
   * This work is licensed under the terms of the GNU GPL, version 2.
   */
 
+#include <asm/page.h>
+
  #define UL(x) _AC(x, UL)
 
+#define PGTABLE_LEVEL_SHIFT(n)	((PAGE_SHIFT - 3) * (4 - (n)) + 3)
  #define PTRS_PER_PTE		(1 << (PAGE_SHIFT - 3))
 
+#if PGTABLE_LEVELS > 2
+#define PMD_SHIFT		PGTABLE_LEVEL_SHIFT(2)
+#define PTRS_PER_PMD		PTRS_PER_PTE
+#define PMD_SIZE		(UL(1) << PMD_SHIFT)
+#define PMD_MASK		(~(PMD_SIZE-1))
+#endif
+
+#if PGTABLE_LEVELS > 3
+#define PUD_SHIFT		PGTABLE_LEVEL_SHIFT(1)
+#define PTRS_PER_PUD		PTRS_PER_PTE
+#define PUD_SIZE		(UL(1) << PUD_SHIFT)
+#define PUD_MASK		(~(PUD_SIZE-1))
+#else
+#define PUD_SIZE                PGDIR_SIZE
+#define PUD_MASK                PGDIR_MASK
+#endif
+
+#define PUD_VALID		(_AT(pudval_t, 1) << 0)
+
  /*
   * PGDIR_SHIFT determines the size a top-level page table entry can map
   * (depending on the configuration, this level can be 0, 1 or 2).
   */
-#define PGDIR_SHIFT		((PAGE_SHIFT - 3) * PGTABLE_LEVELS + 3)
+#define PGDIR_SHIFT		PGTABLE_LEVEL_SHIFT(4 - PGTABLE_LEVELS)
  #define PGDIR_SIZE		(_AC(1, UL) << PGDIR_SHIFT)
  #define PGDIR_MASK		(~(PGDIR_SIZE-1))
  #define PTRS_PER_PGD		(1 << (VA_BITS - PGDIR_SHIFT))
 
  #define PGD_VALID		(_AT(pgdval_t, 1) << 0)
 
-/* From include/asm-generic/pgtable-nopmd.h */
-#define PMD_SHIFT		PGDIR_SHIFT
-#define PTRS_PER_PMD		1
-#define PMD_SIZE		(UL(1) << PMD_SHIFT)
-#define PMD_MASK		(~(PMD_SIZE-1))
-
  /*
   * Section address mask and size definitions.
   */
-#define SECTION_SHIFT		PMD_SHIFT
-#define SECTION_SIZE		(_AC(1, UL) << SECTION_SHIFT)
-#define SECTION_MASK		(~(SECTION_SIZE-1))
+#define SECTION_SHIFT          PMD_SHIFT
+#define SECTION_SIZE           (_AC(1, UL) << SECTION_SHIFT)
+#define SECTION_MASK           (~(SECTION_SIZE-1))
 
  /*
   * Hardware page table definitions.
   *
- * Level 1 descriptor (PMD).
+ * Level 0,1,2 descriptor (PGG, PUD and PMD).
   */
  #define PMD_TYPE_MASK		(_AT(pmdval_t, 3) << 0)
  #define PMD_TYPE_FAULT		(_AT(pmdval_t, 0) << 0)
lib/arm64/asm/pgtable.h +66 -3
index e577d9c..c7632ae 100644
@@ -30,10 +30,12 @@
  #define pgtable_pa(x)		((unsigned long)(x))
 
  #define pgd_none(pgd)		(!pgd_val(pgd))
+#define pud_none(pud)		(!pud_val(pud))
  #define pmd_none(pmd)		(!pmd_val(pmd))
  #define pte_none(pte)		(!pte_val(pte))
 
  #define pgd_valid(pgd)		(pgd_val(pgd) & PGD_VALID)
+#define pud_valid(pud)		(pud_val(pud) & PUD_VALID)
  #define pmd_valid(pmd)		(pmd_val(pmd) & PMD_SECT_VALID)
  #define pte_valid(pte)		(pte_val(pte) & PTE_VALID)
 
@@ -52,15 +54,76 @@ static inline pgd_t *pgd_alloc(void)
  	return pgd;
  }
 
-#define pmd_offset(pgd, addr)	((pmd_t *)pgd)
-#define pmd_free(pmd)
-#define pmd_alloc(pgd, addr)	pmd_offset(pgd, addr)
+static inline pud_t *pgd_page_vaddr(pgd_t pgd)
+{
+	return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
+}
+
+static inline pmd_t *pud_page_vaddr(pud_t pud)
+{
+	return pgtable_va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
+}
+
 
  static inline pte_t *pmd_page_vaddr(pmd_t pmd)
  {
  	return pgtable_va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK);
  }
 
+#if PGTABLE_LEVELS > 2
+#define pmd_index(addr)					\
+	(((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
+#define pmd_offset(pud, addr)				\
+	(pud_page_vaddr(*(pud)) + pmd_index(addr))
+#define pmd_free(pmd) free_page(pmd)
+static inline pmd_t *pmd_alloc_one(void)
+{
+	assert(PTRS_PER_PMD * sizeof(pmd_t) == PAGE_SIZE);
+	pmd_t *pmd = alloc_page();
+	return pmd;
+}
+static inline pmd_t *pmd_alloc(pud_t *pud, unsigned long addr)
+{
+        if (pud_none(*pud)) {
+		pud_t entry;
+		pud_val(entry) = pgtable_pa(pmd_alloc_one()) | PMD_TYPE_TABLE;
+		WRITE_ONCE(*pud, entry);
+	}
+	return pmd_offset(pud, addr);
+}
+#else
+#define pmd_offset(pud, addr)  ((pmd_t *)pud)
+#define pmd_free(pmd)
+#define pmd_alloc(pud, addr)   pmd_offset(pud, addr)
+#endif
+
+#if PGTABLE_LEVELS > 3
+#define pud_index(addr)                                 \
+	(((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
+#define pud_offset(pgd, addr)                           \
+	(pgd_page_vaddr(*(pgd)) + pud_index(addr))
+#define pud_free(pud) free_page(pud)
+static inline pud_t *pud_alloc_one(void)
+{
+	assert(PTRS_PER_PMD * sizeof(pud_t) == PAGE_SIZE);
+	pud_t *pud = alloc_page();
+	return pud;
+}
+static inline pud_t *pud_alloc(pgd_t *pgd, unsigned long addr)
+{
+	if (pgd_none(*pgd)) {
+		pgd_t entry;
+		pgd_val(entry) = pgtable_pa(pud_alloc_one()) | PMD_TYPE_TABLE;
+		WRITE_ONCE(*pgd, entry);
+	}
+	return pud_offset(pgd, addr);
+}
+#else
+#define pud_offset(pgd, addr)  ((pud_t *)pgd)
+#define pud_free(pud)
+#define pud_alloc(pgd, addr)   pud_offset(pgd, addr)
+#endif
+
  #define pte_index(addr) \
  	(((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1))
  #define pte_offset(pmd, addr) \
lib/arm/mmu.c +16 -10
index 540a1e8..6d1c75b 100644
@@ -81,7 +81,8 @@ void mmu_disable(void)
  static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
  {
  	pgd_t *pgd = pgd_offset(pgtable, vaddr);
-	pmd_t *pmd = pmd_alloc(pgd, vaddr);
+	pud_t *pud = pud_alloc(pgd, vaddr);
+	pmd_t *pmd = pmd_alloc(pud, vaddr);
  	pte_t *pte = pte_alloc(pmd, vaddr);
 
  	return &pte_val(*pte);
@@ -133,18 +134,20 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t virt_offset,
  			phys_addr_t phys_start, phys_addr_t phys_end,
  			pgprot_t prot)
  {
-	phys_addr_t paddr = phys_start & PGDIR_MASK;
-	uintptr_t vaddr = virt_offset & PGDIR_MASK;
+	phys_addr_t paddr = phys_start & PUD_MASK;
+	uintptr_t vaddr = virt_offset & PUD_MASK;
  	uintptr_t virt_end = phys_end - paddr + vaddr;
  	pgd_t *pgd;
-	pgd_t entry;
+	pud_t *pud;
+	pud_t entry;
 
-	for (; vaddr < virt_end; vaddr += PGDIR_SIZE, paddr += PGDIR_SIZE) {
-		pgd_val(entry) = paddr;
-		pgd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
-		pgd_val(entry) |= pgprot_val(prot);
+	for (; vaddr < virt_end; vaddr += PUD_SIZE, paddr += PUD_SIZE) {
+		pud_val(entry) = paddr;
+		pud_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
+		pud_val(entry) |= pgprot_val(prot);
  		pgd = pgd_offset(pgtable, vaddr);
-		WRITE_ONCE(*pgd, entry);
+		pud = pud_alloc(pgd, vaddr);
+		WRITE_ONCE(*pud, entry);
  		flush_tlb_page(vaddr);
  	}
  }
@@ -207,6 +210,7 @@ unsigned long __phys_to_virt(phys_addr_t addr)
  void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
  {
  	pgd_t *pgd;
+	pud_t *pud;
  	pmd_t *pmd;
  	pte_t *pte;
 
@@ -215,7 +219,9 @@ void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
 
  	pgd = pgd_offset(pgtable, vaddr);
  	assert(pgd_valid(*pgd));
-	pmd = pmd_offset(pgd, vaddr);
+	pud = pud_offset(pgd, vaddr);
+	assert(pud_valid(*pud));
+	pmd = pmd_offset(pud, vaddr);
  	assert(pmd_valid(*pmd));
 
  	if (pmd_huge(*pmd)) {
arm/cstart64.S +11 -1
index ffdd49f..530ffb6 100644
@@ -157,6 +157,16 @@ halt:
   */
  #define MAIR(attr, mt) ((attr) << ((mt) * 8))
 
+#if PAGE_SHIFT == 16
+#define TCR_TG_FLAGS	TCR_TG0_64K | TCR_TG1_64K
+#elif PAGE_SHIFT == 14
+#define TCR_TG_FLAGS	TCR_TG0_16K | TCR_TG1_16K
+#elif PAGE_SHIFT == 12
+#define TCR_TG_FLAGS	TCR_TG0_4K | TCR_TG1_4K
+#else
+#error Unsupported PAGE_SHIFT
+#endif
+
  .globl asm_mmu_enable
  asm_mmu_enable:
  	tlbi	vmalle1			// invalidate I + D TLBs
@@ -164,7 +174,7 @@ asm_mmu_enable:
 
  	/* TCR */
  	ldr	x1, =TCR_TxSZ(VA_BITS) |		\
-		     TCR_TG0_64K | TCR_TG1_64K |	\
+		     TCR_TG_FLAGS  |			\
  		     TCR_IRGN_WBWA | TCR_ORGN_WBWA |	\
  		     TCR_SHARED
  	mrs	x2, id_aa64mmfr0_el1
#4 lore raw

Hi Nikos,

Quoted 26 lines · Nikos Nikoleris

Hi Alex,

Quoted 5 lines · Alexandru Elisei

Hi Nikos,

I scanned the patches, will try to do a proper review next week. Some suggestions below.

Thanks for the review!

Quoted 9 lines · Nikos Nikoleris
Quoted 5 lines

Make the translation granule configurable for arm64. arm64 supports page sizes of 4K, 16K and 64K. By default, arm64 is configured with 64K pages. configure has been extended with a new argument:

  --page-shift=(12|14|16)

How about --page-size=4K/16K/64K, which accepts lower and upper case 'k'?

Quoted 1 line

It might be just me, because I'm not familiar with memory management,

but each

Quoted 4 lines

time I see page-shift I need to do the math in my head to get to the page size. I think page-size is more intuitive. It's also somewhat similar to the kernel config (CONFIG_ARM64_4K_PAGES, for example, and not CONFIG_ARM64_PAGE_SHIFT_12).

Quoted 4 lines

You're right setting the page size is more intiutive. I had started implementing it that way but I got worried about the added complexity. I tried it again now and it doesn't look that bad. I'll post a v2 that allows you to configure the page size instead.

One option to keep the change as unintrusive as possible would be to keep #define CONFIG_PAGE_SHIFT ${page_shift} in the configure script, but set the variable $page_shift based on the $page-size parameter given by the user.

Thanks,

Alex

Quoted 440 lines
Quoted 6 lines

I might have missed it in the patch, how about printing an error message when the configured page size is not supported by the hardware? kvm-unit-tests runs C code before creating the page tables, and the UART is available very early, so it shouldn't be too hard. We can put the check in setup_mmu().

It's not there and it makes sense to check. Right now if we configure an unsupported page size, tests appear to be running just fine - you only find out when they time out.

Thanks,

Nikos

Quoted 425 lines

Thanks,
Alex

Quoted 423 lines

which allows the user to set the page shift and therefore the page size for arm64. Using the --page-shift for any other architecture results an error message.

To allow for smaller page sizes and 42b VA, this change adds support for 4-level and 3-level page tables. At compile time, we determine how many levels in the page tables we needed.

---
  configure                     | 21 +++++++++++ lib/arm/asm/page.h            |  4 ++ lib/arm/asm/pgtable-hwdef.h   |  4 ++ lib/arm/asm/pgtable.h         |  6 +++ lib/arm/asm/thread_info.h     |  4 +-
  lib/arm64/asm/page.h          | 17 ++++++--- lib/arm64/asm/pgtable-hwdef.h | 38 +++++++++++++------ lib/arm64/asm/pgtable.h       | 69 +++++++++++++++++++++++++++++++++-- lib/arm/mmu.c                 | 26 ++++++++----- arm/cstart64.S                | 12 +++++-

  10 files changed, 169 insertions(+), 32 deletions(-)
configure +0 -0
index 706aab5..94637ec 100755
@@ -25,6 +25,7 @@ vmm="qemu"

  errata_force=0
  erratatxt="$srcdir/errata.txt"
  host_key_document=
+page_shift=
    usage() {
      cat <<-EOF

configure +0 -0
@@ -105,6 +106,9 @@ while [[ "$1" = -* ]]; do
       --host-key-document)

          host_key_document="$arg"
          ;;
+        --page-shift)
+            page_shift="$arg"
+            ;;
      --help)
          usage
          ;;

configure +0 -0
@@ -123,6 +127,22 @@ arch_name=$arch

  [ "$arch" = "aarch64" ] && arch="arm64"
  [ "$arch_name" = "arm64" ] && arch_name="aarch64" +if [ -z "$page_shift" ]; then
+    [ "$arch" = "arm64" ] && page_shift="16" +    [ "$arch" = "arm" ] && page_shift="12"

configure +13 -0
+else
+    if [ "$arch" != "arm64" ]; then
+        echo "--page-shift is not supported for $arch"
+        usage
+    fi
+
+    if [ "$page_shift" != "12" ] && [ "$page_shift" != "14" ] &&
+           [ "$page_shift" != "16" ]; then
+        echo "Page shift of $page_shift not supported for arm64"
+        usage
+    fi
+fi
+

  [ -z "$processor" ] && processor="$arch"
    if [ "$processor" = "arm64" ]; then

configure +1 -0
@@ -254,6 +274,7 @@ cat <<EOF >> lib/config.h
     #define CONFIG_UART_EARLY_BASE ${arm_uart_early_addr}
   #define CONFIG_ERRATA_FORCE ${errata_force}
+#define CONFIG_PAGE_SHIFT ${page_shift}

    EOF
  fi

lib/arm/asm/page.h +0 -0
index 039c9f7..ae0ac2c 100644
@@ -29,6 +29,10 @@ typedef struct { pteval_t pgprot; } pgprot_t;
   #define pgd_val(x)        ((x).pgd)
   #define pgprot_val(x)        ((x).pgprot)

  +/* For compatibility with arm64 page tables */

lib/arm/asm/page.h +3 -0
+#define pud_t pgd_t
+#define pud_val(x) pgd_val(x)
+

  #define __pte(x)        ((pte_t) { (x) } )
  #define __pmd(x)        ((pmd_t) { (x) } )
  #define __pgd(x)        ((pgd_t) { (x) } )

lib/arm/asm/pgtable-hwdef.h +0 -0
index 4107e18..fe1d854 100644
@@ -19,6 +19,10 @@
   #define PTRS_PER_PTE        512
   #define PTRS_PER_PMD        512

  +/* For compatibility with arm64 page tables */

lib/arm/asm/pgtable-hwdef.h +3 -0
+#define PUD_SIZE        PGDIR_SIZE
+#define PUD_MASK        PGDIR_MASK
+

  #define PMD_SHIFT        21
  #define PMD_SIZE        (_AC(1,UL) << PMD_SHIFT) #define PMD_MASK        (~((1 << PMD_SHIFT) - 1))

lib/arm/asm/pgtable.h +0 -0
index 078dd16..4759d82 100644
@@ -53,6 +53,12 @@ static inline pmd_t *pgd_page_vaddr(pgd_t pgd)
       return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
   }

  +/* For compatibility with arm64 page tables */

lib/arm/asm/pgtable.h +5 -0
+#define pud_valid(pud)        pgd_valid(pud)
+#define pud_offset(pgd, addr)  ((pud_t *)pgd)
+#define pud_free(pud)
+#define pud_alloc(pgd, addr)   pud_offset(pgd, addr)
+

  #define pmd_index(addr) \
      (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) #define pmd_offset(pgd, addr) \

lib/arm/asm/thread_info.h +3 -1
index 80ab395..eaa7258 100644
@@ -14,10 +14,12 @@
   #define THREAD_SHIFT        PAGE_SHIFT
   #define THREAD_SIZE        PAGE_SIZE
   #define THREAD_MASK        PAGE_MASK
+#define THREAD_ALIGNMENT    PAGE_SIZE
   #else
   #define THREAD_SHIFT        MIN_THREAD_SHIFT
   #define THREAD_SIZE        (_AC(1,UL) << THREAD_SHIFT)
   #define THREAD_MASK        (~(THREAD_SIZE-1))
+#define THREAD_ALIGNMENT    THREAD_SIZE
   #endif
     #ifndef __ASSEMBLY__
@@ -38,7 +40,7 @@
     static inline void *thread_stack_alloc(void)
   {
-    void *sp = memalign(PAGE_SIZE, THREAD_SIZE);
+    void *sp = memalign(THREAD_ALIGNMENT, THREAD_SIZE);
       return sp + THREAD_START_SP;
   }

  diff --git a/lib/arm64/asm/page.h b/lib/arm64/asm/page.h index 46af552..726a0c0 100644

lib/arm64/asm/page.h +0 -0
@@ -10,38 +10,43 @@
    * This work is licensed under the terms of the GNU GPL, version 2.
    */

  +#include <config.h>
  #include <linux/const.h>
  -#define PGTABLE_LEVELS        2
  #define VA_BITS            42
  -#define PAGE_SHIFT        16
+#define PAGE_SHIFT        CONFIG_PAGE_SHIFT
  #define PAGE_SIZE        (_AC(1,UL) << PAGE_SHIFT) #define PAGE_MASK        (~(PAGE_SIZE-1))
  +#define PGTABLE_LEVELS        (((VA_BITS) - 4) / (PAGE_SHIFT - 3)) +
  #ifndef __ASSEMBLY__
    #define PAGE_ALIGN(addr)    ALIGN(addr, PAGE_SIZE) typedef u64 pteval_t;
  typedef u64 pmdval_t;
+typedef u64 pudval_t;
  typedef u64 pgdval_t;
  typedef struct { pteval_t pte; } pte_t;

lib/arm64/asm/page.h +2 -0
+typedef struct { pmdval_t pmd; } pmd_t;
+typedef struct { pudval_t pud; } pud_t;

  typedef struct { pgdval_t pgd; } pgd_t;
  typedef struct { pteval_t pgprot; } pgprot_t;
    #define pte_val(x)        ((x).pte)

lib/arm64/asm/page.h +2 -0
+#define pmd_val(x)        ((x).pmd)
+#define pud_val(x)        ((x).pud)

  #define pgd_val(x)        ((x).pgd)
  #define pgprot_val(x)        ((x).pgprot)
    #define __pte(x)        ((pte_t) { (x) } )

lib/arm64/asm/page.h +2 -0
+#define __pmd(x)        ((pmd_t) { (x) } )
+#define __pud(x)        ((pud_t) { (x) } )

  #define __pgd(x)        ((pgd_t) { (x) } ) #define __pgprot(x)        ((pgprot_t) { (x) } ) -typedef struct { pgd_t pgd; } pmd_t;

lib/arm64/asm/page.h +0 -3
-#define pmd_val(x)        (pgd_val((x).pgd))
-#define __pmd(x)        ((pmd_t) { __pgd(x) } )
-

  #define __va(x)            ((void *)__phys_to_virt((phys_addr_t)(x))) #define __pa(x)            __virt_to_phys((unsigned long)(x)) diff --git a/lib/arm64/asm/pgtable-hwdef.h b/lib/arm64/asm/pgtable-hwdef.h index 3352489..f9110e1 100644

lib/arm64/asm/pgtable-hwdef.h +0 -0
@@ -9,38 +9,54 @@
    * This work is licensed under the terms of the GNU GPL, version 2.
    */

  +#include <asm/page.h>
+
  #define UL(x) _AC(x, UL)
  +#define PGTABLE_LEVEL_SHIFT(n)    ((PAGE_SHIFT - 3) * (4 - (n)) + 3) #define PTRS_PER_PTE        (1 << (PAGE_SHIFT - 3))
  +#if PGTABLE_LEVELS > 2

lib/arm64/asm/pgtable-hwdef.h +18 -0
+#define PMD_SHIFT        PGTABLE_LEVEL_SHIFT(2)
+#define PTRS_PER_PMD        PTRS_PER_PTE
+#define PMD_SIZE        (UL(1) << PMD_SHIFT)
+#define PMD_MASK        (~(PMD_SIZE-1))
+#endif
+
+#if PGTABLE_LEVELS > 3
+#define PUD_SHIFT        PGTABLE_LEVEL_SHIFT(1)
+#define PTRS_PER_PUD        PTRS_PER_PTE
+#define PUD_SIZE        (UL(1) << PUD_SHIFT)
+#define PUD_MASK        (~(PUD_SIZE-1))
+#else
+#define PUD_SIZE                PGDIR_SIZE
+#define PUD_MASK                PGDIR_MASK
+#endif
+
+#define PUD_VALID        (_AT(pudval_t, 1) << 0)
+

  /*
   * PGDIR_SHIFT determines the size a top-level page table entry can map
   * (depending on the configuration, this level can be 0, 1 or 2). */

lib/arm64/asm/pgtable-hwdef.h +1 -1
-#define PGDIR_SHIFT        ((PAGE_SHIFT - 3) * PGTABLE_LEVELS + 3)
+#define PGDIR_SHIFT        PGTABLE_LEVEL_SHIFT(4 - PGTABLE_LEVELS)

  #define PGDIR_SIZE        (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK        (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD        (1 << (VA_BITS - PGDIR_SHIFT)) #define PGD_VALID        (_AT(pgdval_t, 1) << 0) -/* From include/asm-generic/pgtable-nopmd.h */

lib/arm64/asm/pgtable-hwdef.h +0 -5
-#define PMD_SHIFT        PGDIR_SHIFT
-#define PTRS_PER_PMD        1
-#define PMD_SIZE        (UL(1) << PMD_SHIFT)
-#define PMD_MASK        (~(PMD_SIZE-1))
-

  /*
   * Section address mask and size definitions.
   */

lib/arm64/asm/pgtable-hwdef.h +3 -3
-#define SECTION_SHIFT        PMD_SHIFT
-#define SECTION_SIZE        (_AC(1, UL) << SECTION_SHIFT)
-#define SECTION_MASK        (~(SECTION_SIZE-1))
+#define SECTION_SHIFT          PMD_SHIFT
+#define SECTION_SIZE           (_AC(1, UL) << SECTION_SHIFT)
+#define SECTION_MASK           (~(SECTION_SIZE-1))

    /*
   * Hardware page table definitions.
   *
- * Level 1 descriptor (PMD).
+ * Level 0,1,2 descriptor (PGG, PUD and PMD). */
  #define PMD_TYPE_MASK        (_AT(pmdval_t, 3) << 0) #define PMD_TYPE_FAULT        (_AT(pmdval_t, 0) << 0)

lib/arm64/asm/pgtable.h +2 -0
index e577d9c..c7632ae 100644
@@ -30,10 +30,12 @@
   #define pgtable_pa(x)        ((unsigned long)(x))
     #define pgd_none(pgd)        (!pgd_val(pgd))
+#define pud_none(pud)        (!pud_val(pud))
   #define pmd_none(pmd)        (!pmd_val(pmd))
   #define pte_none(pte)        (!pte_val(pte))
     #define pgd_valid(pgd)        (pgd_val(pgd) & PGD_VALID)
+#define pud_valid(pud)        (pud_val(pud) & PUD_VALID)
   #define pmd_valid(pmd)        (pmd_val(pmd) & PMD_SECT_VALID)
   #define pte_valid(pte)        (pte_val(pte) & PTE_VALID)
   @@ -52,15 +54,76 @@ static inline pgd_t *pgd_alloc(void)
       return pgd;
   }

  -#define pmd_offset(pgd, addr)    ((pmd_t *)pgd)

lib/arm64/asm/pgtable.h +10 -2
-#define pmd_free(pmd)
-#define pmd_alloc(pgd, addr)    pmd_offset(pgd, addr)
+static inline pud_t *pgd_page_vaddr(pgd_t pgd)
+{
+    return pgtable_va(pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK);
+}
+
+static inline pmd_t *pud_page_vaddr(pud_t pud)
+{
+    return pgtable_va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
+}
+

    static inline pte_t *pmd_page_vaddr(pmd_t pmd)
  {
      return pgtable_va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK); }
  +#if PGTABLE_LEVELS > 2

lib/arm64/asm/pgtable.h +53 -0
+#define pmd_index(addr)                    \
+    (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
+#define pmd_offset(pud, addr)                \
+    (pud_page_vaddr(*(pud)) + pmd_index(addr))
+#define pmd_free(pmd) free_page(pmd)
+static inline pmd_t *pmd_alloc_one(void)
+{
+    assert(PTRS_PER_PMD * sizeof(pmd_t) == PAGE_SIZE);
+    pmd_t *pmd = alloc_page();
+    return pmd;
+}
+static inline pmd_t *pmd_alloc(pud_t *pud, unsigned long addr)
+{
+        if (pud_none(*pud)) {
+        pud_t entry;
+        pud_val(entry) = pgtable_pa(pmd_alloc_one()) | PMD_TYPE_TABLE;
+        WRITE_ONCE(*pud, entry);
+    }
+    return pmd_offset(pud, addr);
+}
+#else
+#define pmd_offset(pud, addr)  ((pmd_t *)pud)
+#define pmd_free(pmd)
+#define pmd_alloc(pud, addr)   pmd_offset(pud, addr)
+#endif
+
+#if PGTABLE_LEVELS > 3
+#define pud_index(addr)                                 \
+    (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
+#define pud_offset(pgd, addr)                           \
+    (pgd_page_vaddr(*(pgd)) + pud_index(addr))
+#define pud_free(pud) free_page(pud)
+static inline pud_t *pud_alloc_one(void)
+{
+    assert(PTRS_PER_PMD * sizeof(pud_t) == PAGE_SIZE);
+    pud_t *pud = alloc_page();
+    return pud;
+}
+static inline pud_t *pud_alloc(pgd_t *pgd, unsigned long addr)
+{
+    if (pgd_none(*pgd)) {
+        pgd_t entry;
+        pgd_val(entry) = pgtable_pa(pud_alloc_one()) | PMD_TYPE_TABLE;
+        WRITE_ONCE(*pgd, entry);
+    }
+    return pud_offset(pgd, addr);
+}
+#else
+#define pud_offset(pgd, addr)  ((pud_t *)pgd)
+#define pud_free(pud)
+#define pud_alloc(pgd, addr)   pud_offset(pgd, addr)
+#endif
+

  #define pte_index(addr) \
      (((addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) #define pte_offset(pmd, addr) \

lib/arm/mmu.c +12 -8
index 540a1e8..6d1c75b 100644
@@ -81,7 +81,8 @@ void mmu_disable(void)
   static pteval_t *get_pte(pgd_t *pgtable, uintptr_t vaddr)
   {
       pgd_t *pgd = pgd_offset(pgtable, vaddr);
-    pmd_t *pmd = pmd_alloc(pgd, vaddr);
+    pud_t *pud = pud_alloc(pgd, vaddr);
+    pmd_t *pmd = pmd_alloc(pud, vaddr);
       pte_t *pte = pte_alloc(pmd, vaddr);
         return &pte_val(*pte);
@@ -133,18 +134,20 @@ void mmu_set_range_sect(pgd_t *pgtable, uintptr_t
 virt_offset,
               phys_addr_t phys_start, phys_addr_t phys_end,
               pgprot_t prot)
   {
-    phys_addr_t paddr = phys_start & PGDIR_MASK;
-    uintptr_t vaddr = virt_offset & PGDIR_MASK;
+    phys_addr_t paddr = phys_start & PUD_MASK;
+    uintptr_t vaddr = virt_offset & PUD_MASK;
       uintptr_t virt_end = phys_end - paddr + vaddr;
       pgd_t *pgd;
-    pgd_t entry;
+    pud_t *pud;
+    pud_t entry;
   -    for (; vaddr < virt_end; vaddr += PGDIR_SIZE, paddr += PGDIR_SIZE) {
-        pgd_val(entry) = paddr;
-        pgd_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
-        pgd_val(entry) |= pgprot_val(prot);
+    for (; vaddr < virt_end; vaddr += PUD_SIZE, paddr += PUD_SIZE) {
+        pud_val(entry) = paddr;
+        pud_val(entry) |= PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S;
+        pud_val(entry) |= pgprot_val(prot);
           pgd = pgd_offset(pgtable, vaddr);
-        WRITE_ONCE(*pgd, entry);
+        pud = pud_alloc(pgd, vaddr);
+        WRITE_ONCE(*pud, entry);
           flush_tlb_page(vaddr);
       }

  }

lib/arm/mmu.c +1 -0
@@ -207,6 +210,7 @@ unsigned long __phys_to_virt(phys_addr_t addr)
   void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)
   {
       pgd_t *pgd;
+    pud_t *pud;
       pmd_t *pmd;
       pte_t *pte;
   @@ -215,7 +219,9 @@ void mmu_clear_user(pgd_t *pgtable, unsigned long vaddr)

        pgd = pgd_offset(pgtable, vaddr);
      assert(pgd_valid(*pgd));
-    pmd = pmd_offset(pgd, vaddr);
+    pud = pud_offset(pgd, vaddr);
+    assert(pud_valid(*pud));
+    pmd = pmd_offset(pud, vaddr);
      assert(pmd_valid(*pmd));
        if (pmd_huge(*pmd)) {

arm/cstart64.S +0 -0
index ffdd49f..530ffb6 100644
@@ -157,6 +157,16 @@ halt:
    */
   #define MAIR(attr, mt) ((attr) << ((mt) * 8))

  +#if PAGE_SHIFT == 16

arm/cstart64.S +9 -0
+#define TCR_TG_FLAGS    TCR_TG0_64K | TCR_TG1_64K
+#elif PAGE_SHIFT == 14
+#define TCR_TG_FLAGS    TCR_TG0_16K | TCR_TG1_16K
+#elif PAGE_SHIFT == 12
+#define TCR_TG_FLAGS    TCR_TG0_4K | TCR_TG1_4K
+#else
+#error Unsupported PAGE_SHIFT
+#endif
+

  .globl asm_mmu_enable
  asm_mmu_enable:
      tlbi    vmalle1            // invalidate I + D TLBs

arm/cstart64.S +1 -1
@@ -164,7 +174,7 @@ asm_mmu_enable:
         /* TCR */
       ldr    x1, =TCR_TxSZ(VA_BITS) |        \
-             TCR_TG0_64K | TCR_TG1_64K |    \
+             TCR_TG_FLAGS  |            \
                TCR_IRGN_WBWA | TCR_ORGN_WBWA |    \

               TCR_SHARED
      mrs    x2, id_aa64mmfr0_el1