Skip to contentSkip to conversation

Map larger kernels at early init

PATCH Superseded Steve Capper 2 participants 5 messages Nov 23, 2017 – Dec 12, 2017 arm64

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Steve Capper

author
#0 lore raw

v2 cover Map larger kernels at early init

The early pagetable creation code assumes that a single pgd, pud, pmd and pte are sufficient to map the kernel text for MMU bringup. For 16KB granules this is, unfortunately, rarely the case. Some kernels may be too big even for a 64KB granule employing this scheme.

This patch series addresses the problem in two steps: 1) re-order the reserved_ttbr0 to allow its address computation to be independent of swapper_pg_dir size, 2) re-write the early pgtable code to allow for multiple page table entries at each level.

Changes in v2: Ack added to patch #1, KASLR space calculation redone in patch #2.

Steve Capper (2):
arm64: Re-order reserved_ttbr0 in linker script
arm64: Extend early page table code to allow for larger kernels

 arch/arm64/include/asm/asm-uaccess.h    |   6 +-
 arch/arm64/include/asm/kernel-pgtable.h |  47 +++++++++-
 arch/arm64/include/asm/pgtable.h        |   2 +-
 arch/arm64/include/asm/uaccess.h        |   2 +-
 arch/arm64/kernel/head.S                | 148 +++++++++++++++++++++++---------
 arch/arm64/kernel/vmlinux.lds.S         |   6 +-
 arch/arm64/mm/mmu.c                     |   3 +-
 7 files changed, 164 insertions(+), 50 deletions(-)
Signature · 1 line
2.11.0
author
#1 lore raw

PATCH v2 1/2 arm64: Re-order reserved_ttbr0 in linker script

Currently one resolves the location of the reserved_ttbr0 for PAN by taking a positive offset from swapper_pg_dir. In a future patch we wish to extend the swapper s.t. its size is determined at link time rather than compile time, rendering SWAPPER_DIR_SIZE unsuitable for such a low level calculation.

In this patch we re-arrange the order of the linker script s.t. instead one computes reserved_ttbr0 by subtracting RESERVED_TTBR0_SIZE from swapper_pg_dir.

Signed-off-by Steve Capper <[email protected]>
Acked-by Mark Rutland <[email protected]>
---
 arch/arm64/include/asm/asm-uaccess.h | 6 +++---
 arch/arm64/include/asm/uaccess.h     | 2 +-
 arch/arm64/kernel/vmlinux.lds.S      | 5 ++---
 3 files changed, 6 insertions(+), 7 deletions(-)
arch/arm64/include/asm/asm-uaccess.h +3 -3
index b3da6c886835..e09f02cd3e7a 100644
@@ -12,9 +12,9 @@
  */
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	.macro	__uaccess_ttbr0_disable, tmp1
-	mrs	\tmp1, ttbr1_el1		// swapper_pg_dir
-	add	\tmp1, \tmp1, #SWAPPER_DIR_SIZE	// reserved_ttbr0 at the end of swapper_pg_dir
-	msr	ttbr0_el1, \tmp1		// set reserved TTBR0_EL1
+	mrs	\tmp1, ttbr1_el1			// swapper_pg_dir
+	sub	\tmp1, \tmp1, #RESERVED_TTBR0_SIZE	// reserved_ttbr0 just before swapper_pg_dir
+	msr	ttbr0_el1, \tmp1			// set reserved TTBR0_EL1
 	isb
 	.endm
arch/arm64/include/asm/uaccess.h +1 -1
index fc0f9eb66039..66170fd4b58f 100644
@@ -108,7 +108,7 @@ static inline void __uaccess_ttbr0_disable(void)
 	unsigned long ttbr;
 
 	/* reserved_ttbr0 placed at the end of swapper_pg_dir */
-	ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
+	ttbr = read_sysreg(ttbr1_el1) - RESERVED_TTBR0_SIZE;
 	write_sysreg(ttbr, ttbr0_el1);
 	isb();
 }
arch/arm64/kernel/vmlinux.lds.S +2 -3
index 7da3e5c366a0..2a9475733e84 100644
@@ -206,13 +206,12 @@ SECTIONS
 	. = ALIGN(PAGE_SIZE);
 	idmap_pg_dir = .;
 	. += IDMAP_DIR_SIZE;
-	swapper_pg_dir = .;
-	. += SWAPPER_DIR_SIZE;
-
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
 	reserved_ttbr0 = .;
 	. += RESERVED_TTBR0_SIZE;
 #endif
+	swapper_pg_dir = .;
+	. += SWAPPER_DIR_SIZE;
 
 	__pecoff_data_size = ABSOLUTE(. - __initdata_begin);
 	_end = .;
Signature · 1 line
2.11.0
#2 lore raw

PATCH v2 1/2 arm64: Re-order reserved_ttbr0 in linker script

Quoted 40 lines · Steve Capper

Currently one resolves the location of the reserved_ttbr0 for PAN by taking a positive offset from swapper_pg_dir. In a future patch we wish to extend the swapper s.t. its size is determined at link time rather than compile time, rendering SWAPPER_DIR_SIZE unsuitable for such a low level calculation.

In this patch we re-arrange the order of the linker script s.t. instead one computes reserved_ttbr0 by subtracting RESERVED_TTBR0_SIZE from swapper_pg_dir.

Signed-off-by Steve Capper <[email protected]>
Acked-by Mark Rutland <[email protected]>
---
 arch/arm64/include/asm/asm-uaccess.h | 6 +++---
 arch/arm64/include/asm/uaccess.h     | 2 +-
 arch/arm64/kernel/vmlinux.lds.S      | 5 ++---
 3 files changed, 6 insertions(+), 7 deletions(-)
arch/arm64/include/asm/asm-uaccess.h +3 -3
index b3da6c886835..e09f02cd3e7a 100644
@@ -12,9 +12,9 @@
  */
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
        .macro  __uaccess_ttbr0_disable, tmp1
-       mrs     \tmp1, ttbr1_el1                // swapper_pg_dir
-       add     \tmp1, \tmp1, #SWAPPER_DIR_SIZE // reserved_ttbr0 at the end of swapper_pg_dir
-       msr     ttbr0_el1, \tmp1                // set reserved TTBR0_EL1
+       mrs     \tmp1, ttbr1_el1                        // swapper_pg_dir
+       sub     \tmp1, \tmp1, #RESERVED_TTBR0_SIZE      // reserved_ttbr0 just before swapper_pg_dir
+       msr     ttbr0_el1, \tmp1                        // set reserved TTBR0_EL1
        isb
        .endm
arch/arm64/include/asm/uaccess.h +0 -0
index fc0f9eb66039..66170fd4b58f 100644

arch/arm64/include/asm/uaccess.h static inline void __uaccess_ttbr0_disable(void) · line 110

Quoted 3 lines
arch/arm64/include/asm/uaccess.h +0 -0
@@ -108,7 +108,7 @@ static inline void __uaccess_ttbr0_disable(void)
        unsigned long ttbr;
 
        /* reserved_ttbr0 placed at the end of swapper_pg_dir */

You missed a comment here ^^^

Quoted 15 lines
arch/arm64/include/asm/uaccess.h +1 -1
-       ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
+       ttbr = read_sysreg(ttbr1_el1) - RESERVED_TTBR0_SIZE;
        write_sysreg(ttbr, ttbr0_el1);
        isb();
 }
arch/arm64/kernel/vmlinux.lds.S +0 -2
index 7da3e5c366a0..2a9475733e84 100644
@@ -206,13 +206,12 @@ SECTIONS
        . = ALIGN(PAGE_SIZE);
        idmap_pg_dir = .;
        . += IDMAP_DIR_SIZE;
-       swapper_pg_dir = .;
-       . += SWAPPER_DIR_SIZE;

arch/arm64/kernel/vmlinux.lds.S SECTIONS · line 217

Quoted 10 lines
arch/arm64/kernel/vmlinux.lds.S +2 -1
-
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
        reserved_ttbr0 = .;
        . += RESERVED_TTBR0_SIZE;
 #endif
+       swapper_pg_dir = .;
+       . += SWAPPER_DIR_SIZE;
 
        __pecoff_data_size = ABSOLUTE(. - __initdata_begin);
        _end = .;

With that fixed,

Reviewed-by Ard Biesheuvel <[email protected]>
author
#3 lore raw

PATCH v2 1/2 arm64: Re-order reserved_ttbr0 in linker script

Quoted 43 lines · Ard Biesheuvel
Quoted 42 lines · Steve Capper

Currently one resolves the location of the reserved_ttbr0 for PAN by taking a positive offset from swapper_pg_dir. In a future patch we wish to extend the swapper s.t. its size is determined at link time rather than compile time, rendering SWAPPER_DIR_SIZE unsuitable for such a low level calculation.

In this patch we re-arrange the order of the linker script s.t. instead one computes reserved_ttbr0 by subtracting RESERVED_TTBR0_SIZE from swapper_pg_dir.

Signed-off-by Steve Capper <[email protected]>
Acked-by Mark Rutland <[email protected]>
---
 arch/arm64/include/asm/asm-uaccess.h | 6 +++---
 arch/arm64/include/asm/uaccess.h     | 2 +-
 arch/arm64/kernel/vmlinux.lds.S      | 5 ++---
 3 files changed, 6 insertions(+), 7 deletions(-)
arch/arm64/include/asm/asm-uaccess.h +3 -3
index b3da6c886835..e09f02cd3e7a 100644
@@ -12,9 +12,9 @@
  */
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
        .macro  __uaccess_ttbr0_disable, tmp1
-       mrs     \tmp1, ttbr1_el1                // swapper_pg_dir
-       add     \tmp1, \tmp1, #SWAPPER_DIR_SIZE // reserved_ttbr0 at the end of swapper_pg_dir
-       msr     ttbr0_el1, \tmp1                // set reserved TTBR0_EL1
+       mrs     \tmp1, ttbr1_el1                        // swapper_pg_dir
+       sub     \tmp1, \tmp1, #RESERVED_TTBR0_SIZE      // reserved_ttbr0 just before swapper_pg_dir
+       msr     ttbr0_el1, \tmp1                        // set reserved TTBR0_EL1
        isb
        .endm
arch/arm64/include/asm/uaccess.h +0 -0
index fc0f9eb66039..66170fd4b58f 100644
@@ -108,7 +108,7 @@ static inline void __uaccess_ttbr0_disable(void)
        unsigned long ttbr;
Quoted 4 lines

arch/arm64/include/asm/uaccess.h static inline void __uaccess_ttbr0_disable(void) · line 110

Quoted 1 line
arch/arm64/include/asm/uaccess.h +0 -0
        /* reserved_ttbr0 placed at the end of swapper_pg_dir */

You missed a comment here ^^^

Thanks, I will update this.

Quoted 23 lines
Quoted 23 lines
arch/arm64/include/asm/uaccess.h +1 -1
-       ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
+       ttbr = read_sysreg(ttbr1_el1) - RESERVED_TTBR0_SIZE;
        write_sysreg(ttbr, ttbr0_el1);
        isb();
 }
arch/arm64/kernel/vmlinux.lds.S +2 -3
index 7da3e5c366a0..2a9475733e84 100644
@@ -206,13 +206,12 @@ SECTIONS
        . = ALIGN(PAGE_SIZE);
        idmap_pg_dir = .;
        . += IDMAP_DIR_SIZE;
-       swapper_pg_dir = .;
-       . += SWAPPER_DIR_SIZE;
-
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
        reserved_ttbr0 = .;
        . += RESERVED_TTBR0_SIZE;
 #endif
+       swapper_pg_dir = .;
+       . += SWAPPER_DIR_SIZE;
Quoted 6 lines

arch/arm64/kernel/vmlinux.lds.S SECTIONS · line 217

Quoted 2 lines
arch/arm64/kernel/vmlinux.lds.S +0 -0
        __pecoff_data_size = ABSOLUTE(. - __initdata_begin);
        _end = .;

With that fixed,

Reviewed-by Ard Biesheuvel <[email protected]>

Thanks Ard!

Cheers,
--
Steve
IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.

author
#4 lore raw

PATCH v2 2/2 arm64: Extend early page table code to allow for larger kernels

In reply to Steve Capper

Currently the early assembler page table code assumes that precisely 1xpgd, 1xpud, 1xpmd are sufficient to represent the early kernel text mappings.

Unfortunately this is rarely the case when running with a 16KB granule, and we also run into limits with a 64KB granule when building much larger kernels.

This patch re-writes the early page table logic to compute indices of mappings for each level of page table, and if multiple indices are required, the next-level page table is scaled up accordingly.

Also the required size of the swapper_pg_dir is computed at link time to cover the mapping [KIMAGE_ADDR + VOFFSET, _end]. When KASLR is enabled, an extra page is set aside for each level that may require extra entries at runtime.

Signed-off-by Steve Capper <[email protected]>

---
Changed in v2: KASLR extra space formula corrected. Some of the pre-processor logic simplified.

---
 arch/arm64/include/asm/kernel-pgtable.h |  47 +++++++++-
 arch/arm64/include/asm/pgtable.h        |   2 +-
 arch/arm64/kernel/head.S                | 148 +++++++++++++++++++++++---------
 arch/arm64/kernel/vmlinux.lds.S         |   1 +
 arch/arm64/mm/mmu.c                     |   3 +-
 5 files changed, 158 insertions(+), 43 deletions(-)
arch/arm64/include/asm/kernel-pgtable.h +46 -1
index 7803343e5881..a780f6714b44 100644
@@ -52,7 +52,52 @@
 #define IDMAP_PGTABLE_LEVELS	(ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT))
 #endif
 
-#define SWAPPER_DIR_SIZE	(SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
+
+/*
+ * If KASLR is enabled, then an offset K is added to the kernel address
+ * space. The bottom 21 bits of this offset are zero to guarantee 2MB
+ * alignment for PA and VA.
+ *
+ * For each pagetable level of the swapper, we know that the shift will
+ * be larger than 21 (for the 4KB granule case we use section maps thus
+ * the smallest shift is actually 30) thus there is the possibility that
+ * KASLR can increase the number of pagetable entries by 1, so we make
+ * room for this extra entry.
+ *
+ * Note KASLR cannot increase the number of required entries for a level
+ * by more than one because it increments both the virtual start and end
+ * addresses equally (the extra entry comes from the case where the end
+ * address is just pushed over a boundary and the start address isn't).
+ */
+
+#ifdef CONFIG_RANDOMIZE_BASE
+#define EARLY_KASLR	(1)
+#else
+#define EARLY_KASLR	(0)
+#endif
+
+#define EARLY_ENTRIES(vstart, vend, shift) (((vend) >> (shift)) \
+					- ((vstart) >> (shift)) + 1 + EARLY_KASLR)
+
+#define EARLY_PGDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PGDIR_SHIFT))
+
+#if SWAPPER_PGTABLE_LEVELS > 3
+#define EARLY_PUDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, PUD_SHIFT))
+#else
+#define EARLY_PUDS(vstart, vend) (0)
+#endif
+
+#if SWAPPER_PGTABLE_LEVELS > 2
+#define EARLY_PMDS(vstart, vend) (EARLY_ENTRIES(vstart, vend, SWAPPER_TABLE_SHIFT))
+#else
+#define EARLY_PMDS(vstart, vend) (0)
+#endif
+
+#define EARLY_PAGES(vstart, vend) ( 1 			/* PGDIR page */				\
+			+ EARLY_PGDS((vstart), (vend)) 	/* each PGDIR needs a next level page table */	\
+			+ EARLY_PUDS((vstart), (vend))	/* each PUD needs a next level page table */	\
+			+ EARLY_PMDS((vstart), (vend)))	/* each PMD needs a next level page table */
+#define SWAPPER_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR + TEXT_OFFSET, _end))
 #define IDMAP_DIR_SIZE		(IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
 
 #ifdef CONFIG_ARM64_SW_TTBR0_PAN
arch/arm64/include/asm/pgtable.h +1 -1
index b46e54c2399b..142697e4ba3e 100644
@@ -667,7 +667,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,
 
 extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
 extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
-
+extern pgd_t swapper_pg_end[];
 /*
  * Encode and decode a swap entry:
  *	bits 0-1:	present (must be zero)
arch/arm64/kernel/head.S +108 -40

210 lines not on this page · raw message

arch/arm64/kernel/vmlinux.lds.S +1 -0
index 2a9475733e84..30f230d58124 100644
@@ -212,6 +212,7 @@ SECTIONS
 #endif
 	swapper_pg_dir = .;
 	. += SWAPPER_DIR_SIZE;
+	swapper_pg_end = .;
 
 	__pecoff_data_size = ABSOLUTE(. - __initdata_begin);
 	_end = .;
arch/arm64/mm/mmu.c +2 -1
index f1eb15e0e864..758d276e2851 100644
@@ -612,7 +612,8 @@ void __init paging_init(void)
 	 * allocated with it.
 	 */
 	memblock_free(__pa_symbol(swapper_pg_dir) + PAGE_SIZE,
-		      SWAPPER_DIR_SIZE - PAGE_SIZE);
+		      __pa_symbol(swapper_pg_end) - __pa_symbol(swapper_pg_dir)
+		      - PAGE_SIZE);
 }
 
 /*
Signature · 1 line
2.11.0