arm64: relocatable kernel proof of concept

In preparation of making the kernel relocatable, replace literal symbol references with immediate constants. This is necessary, as the literals will not be populated with meaningful values until after the relocation code has executed.

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm64/kernel/efi-entry.S |  2 +-
 arch/arm64/kernel/head.S      | 36 +++++++++++++++---------------------
 2 files changed, 16 insertions(+), 22 deletions(-)

arch/arm64/kernel/efi-entry.S +0 -0

index 8ce9b0577442..f78e6a1de825 100644

arch/arm64/kernel/efi-entry.S +0 -0

 	add	x21, x0, x21
 
 	/*

arch/arm64/kernel/head.S +0 -0

index 9c856f2aa7a5..1ea3cd2aba34 100644
@@ -46,11 +46,9 @@
 #error TEXT_OFFSET must be less than 2MB
 #endif

arch/arm64/kernel/head.S +2 -4

 	mov	x27, lr
 
 	/*

@@ -351,8 +349,7 @@ __create_page_tables:
 	 * Create the identity mapping.
 	 */
 	mov	x0, x25				// idmap_pg_dir
-	ldr	x3, =KERNEL_START
-	add	x3, x3, x28			// __pa(KERNEL_START)
+	adr_l	x3, KERNEL_START		// __pa(KERNEL_START)
 
 #ifndef CONFIG_ARM64_VA_BITS_48
 #define EXTRA_SHIFT	(PGDIR_SHIFT + PAGE_SHIFT - 3)

@@ -391,9 +388,8 @@ __create_page_tables:
 #endif
 
 	create_pgd_entry x0, x3, x5, x6
-	ldr	x6, =KERNEL_END
 	mov	x5, x3				// __pa(KERNEL_START)
-	add	x6, x6, x28			// __pa(KERNEL_END)
+	adr_l	x6, KERNEL_END			// __pa(KERNEL_END)
 	create_block_map x0, x7, x3, x5, x6
 
 	/*

arch/arm64/kernel/head.S +0 -4

 	sub	x28, x0, x1			// x28 = PHYS_OFFSET - PAGE_OFFSET
 	add	x24, x2, x28			// x24 = PHYS_OFFSET
 	ret
 ENDPROC(__calc_phys_offset)
 
-	.align 3
-1:	.quad	.
-	.quad	PAGE_OFFSET
-
 /*
  * Exception handling. Something went wrong and we can't proceed. We ought to
  * tell the user, but since we don't have any guarantee that we're even

Signature · 1 line

1.8.3.2

Hi Ard,

Quoted 4 lines · Ard Biesheuvel

In preparation of making the kernel relocatable, replace literal symbol references with immediate constants. This is necessary, as the literals will not be populated with meaningful values until after the relocation code has executed.

The majority of these changes look like nice cleanups/simplifications regardless of whether the kernel is relocatable, so I like most of the patch.

Quoted 6 lines

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm64/kernel/efi-entry.S |  2 +-
 arch/arm64/kernel/head.S      | 36 +++++++++++++++---------------------
 2 files changed, 16 insertions(+), 22 deletions(-)

arch/arm64/kernel/efi-entry.S ENTRY(efi_stub_entry) · line 64

Quoted 10 lines

arch/arm64/kernel/efi-entry.S +1 -1

index 8ce9b0577442..f78e6a1de825 100644
@@ -61,7 +61,7 @@ ENTRY(efi_stub_entry)
       */
      mov     x20, x0         // DTB address
      ldr     x0, [sp, #16]   // relocated _text address
-     ldr     x21, =stext_offset
+     movz    x21, #:abs_g0:stext_offset

This looks a little scary. Why do we need to use a movz with a special relocation rather than a simple mov? As far as I can see mov has sufficient range.

Quoted 11 lines

arch/arm64/kernel/efi-entry.S +0 -0

      add     x21, x0, x21
 
      /*

arch/arm64/kernel/head.S +0 -0

index 9c856f2aa7a5..1ea3cd2aba34 100644
@@ -46,11 +46,9 @@
 #error TEXT_OFFSET must be less than 2MB
 #endif
 

arch/arm64/kernel/head.S line 52

Quoted 9 lines

arch/arm64/kernel/head.S +3 -5

-     .macro  pgtbl, ttb0, ttb1, virt_to_phys
-     ldr     \ttb1, =swapper_pg_dir
-     ldr     \ttb0, =idmap_pg_dir
-     add     \ttb1, \ttb1, \virt_to_phys
-     add     \ttb0, \ttb0, \virt_to_phys
+     .macro  pgtbl, ttb0, ttb1
+     adrp    \ttb1, swapper_pg_dir
+     adrp    \ttb0, idmap_pg_dir
      .endm

I reckon we should just kill pgtbl and use adrp inline in both of the callsites. That way we can also get rid of the comment in each case, beacuse the assembly itself will document which register gets which table address.

arch/arm64/kernel/head.S line 64

Quoted 7 lines

arch/arm64/kernel/head.S +1 -1

 #ifdef CONFIG_ARM64_64K_PAGES

@@ -63,7 +61,7 @@
 #define TABLE_SHIFT  PUD_SHIFT
 #endif
 
-#define KERNEL_START KERNEL_RAM_VADDR
+#define KERNEL_START _text

We can kill the KERNEL_RAM_VADDR definition too, I believe. It's only used here.

arch/arm64/kernel/head.S ENDPROC(stext) · line 323

Quoted 9 lines

arch/arm64/kernel/head.S +1 -1

 #define KERNEL_END   _end
 
 /*

@@ -322,7 +320,7 @@ ENDPROC(stext)
  *     been enabled
  */
 __create_page_tables:
-     pgtbl   x25, x26, x28                   // idmap_pg_dir and swapper_pg_dir addresses
+     pgtbl   x25, x26                        // idmap_pg_dir and swapper_pg_dir addresses

Here we could just have:

adrp x25, idmap_pg_dir
adrp x26, swapper_pg_dir

Quoted 23 lines

arch/arm64/kernel/head.S +2 -4

      mov     x27, lr
 
      /*

@@ -351,8 +349,7 @@ __create_page_tables:
       * Create the identity mapping.
       */
      mov     x0, x25                         // idmap_pg_dir
-     ldr     x3, =KERNEL_START
-     add     x3, x3, x28                     // __pa(KERNEL_START)
+     adr_l   x3, KERNEL_START                // __pa(KERNEL_START)
 
 #ifndef CONFIG_ARM64_VA_BITS_48
 #define EXTRA_SHIFT  (PGDIR_SHIFT + PAGE_SHIFT - 3)

@@ -391,9 +388,8 @@ __create_page_tables:
 #endif
 
      create_pgd_entry x0, x3, x5, x6
-     ldr     x6, =KERNEL_END
      mov     x5, x3                          // __pa(KERNEL_START)
-     add     x6, x6, x28                     // __pa(KERNEL_END)
+     adr_l   x6, KERNEL_END                  // __pa(KERNEL_END)
      create_block_map x0, x7, x3, x5, x6
 

arch/arm64/kernel/head.S __create_page_tables: · line 404

Quoted 10 lines

arch/arm64/kernel/head.S +3 -1

      /*

@@ -402,8 +398,10 @@ __create_page_tables:
      mov     x0, x26                         // swapper_pg_dir
      mov     x5, #PAGE_OFFSET
      create_pgd_entry x0, x5, x3, x6
-     ldr     x6, =KERNEL_END
+     adr_l   x6, KERNEL_END
      mov     x3, x24                         // phys offset
+     sub     x6, x6, x3                      // kernel memsize
+     add     x6, x6, x5                      // __va(KERNEL_END)

I'm not sure on this; it makes it consistent with the rest w.r.t. using adr* but it's now a little harder to read :/

arch/arm64/kernel/head.S ENDPROC(el2_setup) · line 539

Quoted 10 lines

arch/arm64/kernel/head.S +1 -2

      create_block_map x0, x7, x3, x5, x6
 
      /*

@@ -538,8 +536,7 @@ ENDPROC(el2_setup)
  * in x20. See arch/arm64/include/asm/virt.h for more info.
  */
 ENTRY(set_cpu_boot_mode_flag)
-     ldr     x1, =__boot_cpu_mode            // Compute __boot_cpu_mode
-     add     x1, x1, x28
+     adr_l   x1, __boot_cpu_mode             // Compute __boot_cpu_mode

The comment seems redundant now (and arguably was before). I think it's a distraction we can drop.

arch/arm64/kernel/head.S ENTRY(secondary_startup) · line 598

Quoted 9 lines

arch/arm64/kernel/head.S +1 -1

      cmp     w20, #BOOT_CPU_MODE_EL2
      b.ne    1f
      add     x1, x1, #4

@@ -598,7 +595,7 @@ ENTRY(secondary_startup)
      /*
       * Common entry point for secondary CPUs.
       */
-     pgtbl   x25, x26, x28                   // x25=TTBR0, x26=TTBR1
+     pgtbl   x25, x26                        // x25=TTBR0, x26=TTBR1

As mentioned above, I think this is clearer as:

adrp x25, idmap_pg_dir
adrp x26, swapper_pg_dir

Quoted 2 lines

arch/arm64/kernel/head.S +0 -0

      bl      __cpu_setup                     // initialise processor
 

arch/arm64/kernel/head.S ENDPROC(__turn_mmu_on) · line 657

Quoted 10 lines

arch/arm64/kernel/head.S +3 -2

      ldr     x21, =secondary_data

@@ -655,17 +652,14 @@ ENDPROC(__turn_mmu_on)
  * Calculate the start of physical memory.
  */
 __calc_phys_offset:
-     adr     x0, 1f
-     ldp     x1, x2, [x0]
+     adrp    x0, KERNEL_START                // __pa(KERNEL_START)
+     ldr     x1, =KERNEL_RAM_VADDR           // __va(KERNEL_START)
+     mov     x2, PAGE_OFFSET

Missing '#' on the PAGE_OFFSET immediate.

Quoted 16 lines

arch/arm64/kernel/head.S +0 -5

      sub     x28, x0, x1                     // x28 = PHYS_OFFSET - PAGE_OFFSET
      add     x24, x2, x28                    // x24 = PHYS_OFFSET
      ret
 ENDPROC(__calc_phys_offset)
 
-     .align 3
-1:   .quad   .
-     .quad   PAGE_OFFSET
-
 /*
  * Exception handling. Something went wrong and we can't proceed. We ought to
  * tell the user, but since we don't have any guarantee that we're even
--

1.8.3.2

These patches is a proof of concept of how we could potentially relocate the kernel at runtime. This code is rough around the edges, and there are a few unresolved issues, hence this RFC.

With these patches, the kernel can essentially execute at any virtual offset. There are a couple of reasons why we would want this:
- performance: we can align PHYS_OFFSET so that most of the linear mapping can be done using 512 MB or 1 GB blocks (depending on page size), instead of the more granular level that is currently unavoidable if Image cannot be loaded at base of RAM (since PHYS_OFFSET is tied to the start of the kernel Image).

- security: with these changes, we can put the kernel Image anywhere in physical memory, and we can put the physical memory anywhere in the upper half of the virtual address range (modulo alignment). This gives us quite a number of bits of to play with if we were to randomize the kernel virtual address space. Also, this is entirely under the control of the boot loader, which is probably in better shape to get its hands on some entropy than the early kernel boot code.
- convenience: fewer constraints when loading the kernel into memory, as it can execute from anywhere.

How it works:
- an additional boot argument 'image offset' is passed in x1 by the boot loader, which should contain a value that is at least the offset of Image into physical memory. Higher values are also possible, and may be used to randomize the kernel VA space.

- the kernel binary is runtime relocated to PAGE_OFFSET + image offset

Issues:
- Since AArch64 uses the ELF RELA format (where the addends are in the relocation table and not in the code), the relocations need to be applied even if the Image runs from the same offset it was linked at. It also means that some values that are produced by the linker (_kernel_size_le, etc) are missing from the binary. This will probably need a fixup step.
- The module area may be out of range, which needs to be worked around with module PLTs. This is straight forward but I haven't implemented it yet for arm64.
- The core extable is most likely broken, and would need to be changed to use relative offsets instead of absolute addresses.

Hi Ard,

I agree that we want to be able to load the kernel anywhwere in memory (modulo alignment restrictions and the like). However, I'm not keen on the approach taken; I'd rather see the linear mapping split from the text mapping. More on that below.

Quoted 11 lines · Ard Biesheuvel

These patches is a proof of concept of how we could potentially relocate the kernel at runtime. This code is rough around the edges, and there are a few unresolved issues, hence this RFC.

With these patches, the kernel can essentially execute at any virtual offset. There are a couple of reasons why we would want this:
- performance: we can align PHYS_OFFSET so that most of the linear mapping can be done using 512 MB or 1 GB blocks (depending on page size), instead of the more granular level that is currently unavoidable if Image cannot be loaded at base of RAM (since PHYS_OFFSET is tied to the start of the kernel Image).

Isn't this gain somewhat offset by having to build the kernel as a PIE?

If we're doing this for performance it would be good to see numbers.

Quoted 9 lines

- security: with these changes, we can put the kernel Image anywhere in physical memory, and we can put the physical memory anywhere in the upper half of the virtual address range (modulo alignment). This gives us quite a number of bits of to play with if we were to randomize the kernel virtual address space. Also, this is entirely under the control of the boot loader, which is probably in better shape to get its hands on some entropy than the early kernel boot code.
- convenience: fewer constraints when loading the kernel into memory, as it can execute from anywhere.

I agree that making things easier for loaders is for the best.

Quoted 5 lines

How it works:
- an additional boot argument 'image offset' is passed in x1 by the boot loader, which should contain a value that is at least the offset of Image into physical memory. Higher values are also possible, and may be used to randomize the kernel VA space.

I have a very strong suspicion that bootloaders in the wild don't zero x1-x3, and that given that we might not have a reliable mechanism for acquiring the offset.

Quoted 13 lines

- the kernel binary is runtime relocated to PAGE_OFFSET + image offset

Issues:
- Since AArch64 uses the ELF RELA format (where the addends are in the relocation table and not in the code), the relocations need to be applied even if the Image runs from the same offset it was linked at. It also means that some values that are produced by the linker (_kernel_size_le, etc) are missing from the binary. This will probably need a fixup step.
- The module area may be out of range, which needs to be worked around with module PLTs. This is straight forward but I haven't implemented it yet for arm64.
- The core extable is most likely broken, and would need to be changed to use relative offsets instead of absolute addresses.

This sounds like it's going to be a big headache.

I'd rather see that we decouple the kernel (text/data) mapping from the linear mapping, with the former given a fixed VA independent of the PA of the kernel Image (which would still need to be at a 2M-aligned address + text_offset, and not straddling a 512M boundary).

That would allow us to place the kernel anywhere in memory (modulo those constraints), enable us to address memory below the kernel when we do so, and would still allow the kernel to be built with absolute addressing, which keeps things simple and fast.

That doesn't give us VA randomisation, but that could be built atop by reserving a larger VA range than necessary for the kernel, and have the kernel pick a window from within that (assuming we can find some entropy early on) to relocate itself to. That would also be independent of the physical layout, which is nice -- we could have randomised VAs even with a trivial loader that always placed the kernel at the same address (which is likely to be the common case).

When I looked at this a while back it seemed like the majority of the changes were fairly mechanical (introducing and using text_to_phys/phys_to_text and leaving virt_to_x for the linear mapping), and the big pain points seemed to be the page table init (where we rely on memory at the end of the kernel mapping) and KVM.

Quoted 7 lines

Quoted 5 lines

- performance: we can align PHYS_OFFSET so that most of the linear mapping can be done using 512 MB or 1 GB blocks (depending on page size), instead of the more granular level that is currently unavoidable if Image cannot be loaded at base of RAM (since PHYS_OFFSET is tied to the start of the kernel Image).

Isn't this gain somewhat offset by having to build the kernel as a PIE?

I don't think so. Note that this is not -fpic code, it's just the ld option that dumps the reloc and dynsym tables into the output image. The reloc penalty is boottime only.

Quoted 4 lines

I have a very strong suspicion that bootloaders in the wild don't zero x1-x3, and that given that we might not have a reliable mechanism for acquiring the offset.

OK, sounds about time to start complaining about that then.

Quoted 12 lines

Quoted 11 lines

Issues:
- Since AArch64 uses the ELF RELA format (where the addends are in the relocation table and not in the code), the relocations need to be applied even if the Image runs from the same offset it was linked at. It also means that some values that are produced by the linker (_kernel_size_le, etc) are missing from the binary. This will probably need a fixup step.
- The module area may be out of range, which needs to be worked around with module PLTs. This is straight forward but I haven't implemented it yet for arm64.
- The core extable is most likely broken, and would need to be changed to use relative offsets instead of absolute addresses.

Quoted 2 lines

This sounds like it's going to be a big headache.

It's all manageable, really. The module PLT thing is something I already implemented for 32-bit ARM here:
http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/305539.html
(only Russell couldn't be bothered to merge it)

The extable is already relative on x86, and the fixup step is some straight forward ELF mangling on vmlinux before performing the objcopy.
But yes, it's rather ugly.

Quoted 5 lines

I'd rather see that we decouple the kernel (text/data) mapping from the linear mapping, with the former given a fixed VA independent of the PA of the kernel Image (which would still need to be at a 2M-aligned address + text_offset, and not straddling a 512M boundary).

Hmm, that's quite nice, actually, It also fixes the module range problem, and for VA randomization we could move both regions together.

Quoted 11 lines

Quoted 7 lines

Quoted 5 lines

- performance: we can align PHYS_OFFSET so that most of the linear mapping can be done using 512 MB or 1 GB blocks (depending on page size), instead of the more granular level that is currently unavoidable if Image cannot be loaded at base of RAM (since PHYS_OFFSET is tied to the start of the kernel Image).

Isn't this gain somewhat offset by having to build the kernel as a PIE?

I don't think so. Note that this is not -fpic code, it's just the ld option that dumps the reloc and dynsym tables into the output image. The reloc penalty is boottime only.

Ah, ok.

Quoted 5 lines

Quoted 3 lines

I have a very strong suspicion that bootloaders in the wild don't zero x1-x3, and that given that we might not have a reliable mechanism for acquiring the offset.

OK, sounds about time to start complaining about that then.

I guess so.

Quoted 19 lines

Quoted 12 lines

Quoted 11 lines

Issues:
- Since AArch64 uses the ELF RELA format (where the addends are in the relocation table and not in the code), the relocations need to be applied even if the Image runs from the same offset it was linked at. It also means that some values that are produced by the linker (_kernel_size_le, etc) are missing from the binary. This will probably need a fixup step.
- The module area may be out of range, which needs to be worked around with module PLTs. This is straight forward but I haven't implemented it yet for arm64.
- The core extable is most likely broken, and would need to be changed to use relative offsets instead of absolute addresses.

Quoted 1 line

This sounds like it's going to be a big headache.

It's all manageable, really. The module PLT thing is something I already implemented for 32-bit ARM here:
http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/305539.html
(only Russell couldn't be bothered to merge it)

Quoted 4 lines

The extable is already relative on x86, and the fixup step is some straight forward ELF mangling on vmlinux before performing the objcopy.
But yes, it's rather ugly.

Hmm. I'd be rather worried about the fixup step; I suspect that'll be fragile and rarely tested. Perhaps we could verify them at boot time?

Quoted 8 lines

Quoted 5 lines

I'd rather see that we decouple the kernel (text/data) mapping from the linear mapping, with the former given a fixed VA independent of the PA of the kernel Image (which would still need to be at a 2M-aligned address + text_offset, and not straddling a 512M boundary).

Hmm, that's quite nice, actually, It also fixes the module range problem, and for VA randomization we could move both regions together.

Ah, good point. I hadn't consdiered modules all that much, but it sounds like it could work.

Quoted 2 lines

Hmm. I'd be rather worried about the fixup step; I suspect that'll be fragile and rarely tested. Perhaps we could verify them at boot time?

It would essentially be __relocate_kernel() but executed at build time against vmlinux with a reloc offset of #0

These patches is a proof of concept of how we could potentially relocate the kernel at runtime. This code is rough around the edges, and there are a few unresolved issues, hence this RFC.

With these patches, the kernel can essentially execute at any virtual offset. There are a couple of reasons why we would want this:
- performance: we can align PHYS_OFFSET so that most of the linear mapping can be done using 512 MB or 1 GB blocks (depending on page size), instead of the more granular level that is currently unavoidable if Image cannot be loaded at base of RAM (since PHYS_OFFSET is tied to the start of the kernel Image).
- security: with these changes, we can put the kernel Image anywhere in physical memory, and we can put the physical memory anywhere in the upper half of the virtual address range (modulo alignment). This gives us quite a number of bits of to play with if we were to randomize the kernel virtual address space. Also, this is entirely under the control of the boot loader, which is probably in better shape to get its hands on some entropy than the early kernel boot code.

- convenience: fewer constraints when loading the kernel into memory, as it can execute from anywhere.

How it works:
- an additional boot argument 'image offset' is passed in x1 by the boot loader, which should contain a value that is at least the offset of Image into physical memory. Higher values are also possible, and may be used to randomize the kernel VA space.
- the kernel binary is runtime relocated to PAGE_OFFSET + image offset

Issues:
- Since AArch64 uses the ELF RELA format (where the addends are in the relocation table and not in the code), the relocations need to be applied even if the Image runs from the same offset it was linked at. It also means that some values that are produced by the linker (_kernel_size_le, etc) are missing from the binary. This will probably need a fixup step.

- The module area may be out of range, which needs to be worked around with module PLTs. This is straight forward but I haven't implemented it yet for arm64.
- The core extable is most likely broken, and would need to be changed to use relative offsets instead of absolute addresses.
- Probably lots and lots of other roadblocks, hence this RFC

Output from QEMU/efi with 2 GB of memory:

Base of RAM:
0x000040000000-0x00004000ffff [Loader Data | | | | | |WB|WT|WC|UC]

Physical location of Image:
0x00007f400000-0x00007fe6ffff [Loader Data | | | | | |WB|WT|WC|UC]

Virtual kernel memory layout:

    vmalloc : 0xffffff8000000000 - 0xffffffbdbfff0000   (   246 GB)
    vmemmap : 0xffffffbdc0000000 - 0xffffffbfc0000000   (     8 GB maximum)
              0xffffffbdc1000000 - 0xffffffbdc3000000   (    32 MB actual)
    fixed   : 0xffffffbffa9fe000 - 0xffffffbffac00000   (  2056 KB)
    PCI I/O : 0xffffffbffae00000 - 0xffffffbffbe00000   (    16 MB)
    modules : 0xffffffbffc000000 - 0xffffffc000000000   (    64 MB)
    memory  : 0xffffffc000000000 - 0xffffffc080000000   (  2048 MB)
      .init : 0xffffffc03fb82000 - 0xffffffc03fdbe000   (  2288 KB)
      .text : 0xffffffc03f480000 - 0xffffffc03fb81344   (  7173 KB)
      .data : 0xffffffc03fdc2000 - 0xffffffc03fe28c00   (   411 KB)

Ard Biesheuvel (3):
arm64: head.S: replace early literals with constant immediates
arm64: add support for relocatable kernel
arm64/efi: use relocated kernel

 arch/arm64/Kconfig              |  3 ++
 arch/arm64/Makefile             |  4 ++
 arch/arm64/kernel/efi-entry.S   |  6 ++-
 arch/arm64/kernel/efi-stub.c    |  5 ++-
 arch/arm64/kernel/head.S        | 94 +++++++++++++++++++++++++++++++----------
 arch/arm64/kernel/image.h       |  8 +++-
 arch/arm64/kernel/vmlinux.lds.S | 12 ++++++
 scripts/sortextable.c           |  4 +-
 8 files changed, 107 insertions(+), 29 deletions(-)

--
1.8.3.2

Quoted 4 lines · Ard Biesheuvel

These patches is a proof of concept of how we could potentially relocate the kernel at runtime. This code is rough around the edges, and there are a few unresolved issues, hence this RFC.

Quoted 14 lines

With these patches, the kernel can essentially execute at any virtual offset. There are a couple of reasons why we would want this:
- performance: we can align PHYS_OFFSET so that most of the linear mapping can be done using 512 MB or 1 GB blocks (depending on page size), instead of the more granular level that is currently unavoidable if Image cannot be loaded at base of RAM (since PHYS_OFFSET is tied to the start of the kernel Image).
- security: with these changes, we can put the kernel Image anywhere in physical memory, and we can put the physical memory anywhere in the upper half of the virtual address range (modulo alignment). This gives us quite a number of bits of to play with if we were to randomize the kernel virtual address space. Also, this is entirely under the control of the boot loader, which is probably in better shape to get its hands on some entropy than the early kernel boot code.

This is great! Thank you for working on this. ARM kernel text ASLR has been on my TODO list for a while now, but I kept looking at the missing relocation support and then would start crying. :) (Do you have any plans for 32-bit ARM relocation support?)

Some random brain-dump from my experiences with kASLR on x86:

I don't want to exclusively depend on the bootloader for kASLR. There is, I think, already a way to pass entropy into the kernel from the bootloader, so perhaps that could be used on top of chipset-specific entropy sources? I would like to try to keep the way kASLR gets enabled/used/whatever at least compatible with x86 (which probably means adding some additional knobs to x86 to gain what ARM will have).

Possibly related to running with/detecting an offset, we need a way to communicate that kASLR is active through the compressed kernel to uncompressed kernel. x86 is going to be using x86's setup_data, but we may need to generalize this. (The reasoning here is that when kaslr is disabled at runtime, we should turn off other kernel ASLR, like module offset ASLR, without duplicating kernel command line parameter parsing -- which is what x86 is currently doing now.) Just examining the offset isn't sufficient because perhaps we got randomized to offset 0. :)

For note, here's kernel module offset ASLR for ARM:
https://git.kernel.org/cgit/linux/kernel/git/kees/linux.git/commit/?h=kaslr/arm&id=773234f719a5cccb662840b90f41fad2930b2460

You mention the linear mappings in "performance", which I worry may be at odds with kASLR? Can large mappings still be used even when we've got smaller alignment goals? Since you mention the "upper half of the virtual address range", I assume ARM is built using the -2GB addressing as used by x86, is that right? So it sounds like it would be similar entropy to x86.

We should keep in mind the idea of splitting physical ASLR from virtual ASLR. The virtual addresses remain bound to the -2GB addressing, but the kernel could be anywhere in physical RAM when there is >4GB available. This is currently being worked on for x86.

Quoted 10 lines

- convenience: fewer constraints when loading the kernel into memory, as it can execute from anywhere.

How it works:
- an additional boot argument 'image offset' is passed in x1 by the boot loader, which should contain a value that is at least the offset of Image into physical memory. Higher values are also possible, and may be used to randomize the kernel VA space.
- the kernel binary is runtime relocated to PAGE_OFFSET + image offset

Quoted 6 lines

Issues:
- Since AArch64 uses the ELF RELA format (where the addends are in the relocation table and not in the code), the relocations need to be applied even if the Image runs from the same offset it was linked at. It also means that some values that are produced by the linker (_kernel_size_le, etc) are missing from the binary. This will probably need a fixup step.

I had no end of troubles with linkers changing their behavior. :) Have you looked at arch/x86/tools/relocs* tool, or the reloc mechanisms in arch/x86/boot/compressed/misc.c handle_relocations()? It looks like you're using real ELF relocations in patch 2, which seems more sensible that what x86 is doing (reinventing relocation tables). I'm not sure why it's done that way, though.

Quoted 43 lines

- The module area may be out of range, which needs to be worked around with module PLTs. This is straight forward but I haven't implemented it yet for arm64.
- The core extable is most likely broken, and would need to be changed to use relative offsets instead of absolute addresses.
- Probably lots and lots of other roadblocks, hence this RFC

Output from QEMU/efi with 2 GB of memory:

Base of RAM:
0x000040000000-0x00004000ffff [Loader Data | | | | | |WB|WT|WC|UC]

Physical location of Image:
0x00007f400000-0x00007fe6ffff [Loader Data | | | | | |WB|WT|WC|UC]

Virtual kernel memory layout:

    vmalloc : 0xffffff8000000000 - 0xffffffbdbfff0000   (   246 GB)
    vmemmap : 0xffffffbdc0000000 - 0xffffffbfc0000000   (     8 GB maximum)
              0xffffffbdc1000000 - 0xffffffbdc3000000   (    32 MB actual)
    fixed   : 0xffffffbffa9fe000 - 0xffffffbffac00000   (  2056 KB)
    PCI I/O : 0xffffffbffae00000 - 0xffffffbffbe00000   (    16 MB)
    modules : 0xffffffbffc000000 - 0xffffffc000000000   (    64 MB)
    memory  : 0xffffffc000000000 - 0xffffffc080000000   (  2048 MB)
      .init : 0xffffffc03fb82000 - 0xffffffc03fdbe000   (  2288 KB)
      .text : 0xffffffc03f480000 - 0xffffffc03fb81344   (  7173 KB)
      .data : 0xffffffc03fdc2000 - 0xffffffc03fe28c00   (   411 KB)

Ard Biesheuvel (3):
arm64: head.S: replace early literals with constant immediates
arm64: add support for relocatable kernel
arm64/efi: use relocated kernel

 arch/arm64/Kconfig              |  3 ++
 arch/arm64/Makefile             |  4 ++
 arch/arm64/kernel/efi-entry.S   |  6 ++-
 arch/arm64/kernel/efi-stub.c    |  5 ++-
 arch/arm64/kernel/head.S        | 94 +++++++++++++++++++++++++++++++----------
 arch/arm64/kernel/image.h       |  8 +++-
 arch/arm64/kernel/vmlinux.lds.S | 12 ++++++
 scripts/sortextable.c           |  4 +-
 8 files changed, 107 insertions(+), 29 deletions(-)

Quoted 3 lines

--
1.8.3.2

I'll try to get this series into a state I can test on my hardware. Thanks again!

Quoted 10 lines

Possibly related to running with/detecting an offset, we need a way to communicate that kASLR is active through the compressed kernel to uncompressed kernel. x86 is going to be using x86's setup_data, but we may need to generalize this. (The reasoning here is that when kaslr is disabled at runtime, we should turn off other kernel ASLR, like module offset ASLR, without duplicating kernel command line parameter parsing -- which is what x86 is currently doing now.) Just examining the offset isn't sufficient because perhaps we got randomized to offset 0. :)

There is no decompressor on arm64, just the core kernel Image. So if an offset needs to be chosen before branching into the kernel proper, it needs to be the bootloader that chooses it.

Quoted 7 lines

You mention the linear mappings in "performance", which I worry may be at odds with kASLR? Can large mappings still be used even when we've got smaller alignment goals? Since you mention the "upper half of the virtual address range", I assume ARM is built using the -2GB addressing as used by x86, is that right? So it sounds like it would be similar entropy to x86.

I haven't quantified the performance gain, but it is arguably more efficient to map RAM using 1 GB blocks than using 2 MB sections. On the other part of the question, I really need to do more research on what x86 implements in the first place before even trying to answer it.

Quoted 11 lines

Quoted 10 lines

Possibly related to running with/detecting an offset, we need a way to communicate that kASLR is active through the compressed kernel to uncompressed kernel. x86 is going to be using x86's setup_data, but we may need to generalize this. (The reasoning here is that when kaslr is disabled at runtime, we should turn off other kernel ASLR, like module offset ASLR, without duplicating kernel command line parameter parsing -- which is what x86 is currently doing now.) Just examining the offset isn't sufficient because perhaps we got randomized to offset 0. :)

Quoted 3 lines

There is no decompressor on arm64, just the core kernel Image. So if an offset needs to be chosen before branching into the kernel proper, it needs to be the bootloader that chooses it.

Agreed.

Our equivalent to setup_data is the DT /chosen node, and I don't think we want to try parsing that before we've turned on the MMU.

However, for the UEFI boot case we could have the stub do something more intelligent and choose a random physical offset itself.

Quoted 8 lines

Quoted 7 lines

You mention the linear mappings in "performance", which I worry may be at odds with kASLR? Can large mappings still be used even when we've got smaller alignment goals? Since you mention the "upper half of the virtual address range", I assume ARM is built using the -2GB addressing as used by x86, is that right? So it sounds like it would be similar entropy to x86.

Quoted 5 lines

I haven't quantified the performance gain, but it is arguably more efficient to map RAM using 1 GB blocks than using 2 MB sections. On the other part of the question, I really need to do more research on what x86 implements in the first place before even trying to answer it.

That might not always be true, depending on the TLB implementation (though it's better to assume that it is, as it shouldn't result in a performance loss).

Also, if you use DEBUG_RODATA the kernel won't be mapped with 1GB mappings after early boot anyway.

Quoted 3 lines

Also, if you use DEBUG_RODATA the kernel won't be mapped with 1GB mappings after early boot anyway.

That's not the point. The point is that, since physical memory starts at the base of the kernel image, as far as the kernel is concerned, the relative alignment of the virtual and physical address spaces may result in *all* of memory being mapped using 2 MB sections rather than 1 GB blocks.

arch/arm64/include/asm/memory.h +4 -0

index f800d45ea226..7dfe1b0c9c01 100644
@@ -107,6 +107,10 @@
 #define MT_S2_NORMAL		0xf
 #define MT_S2_DEVICE_nGnRE	0x1
 
+#define __TEXT(x)	((x) & ~(UL(1) << 56))
+#define __VIRT(x)	((x) | (UL(1) << 56))
+#define __IS_TEXT(x)	(!((x) & (UL(1) << 56)))
+
 #ifndef __ASSEMBLY__
 
 extern phys_addr_t		memstart_addr;

@@ -141,9 +145,23 @@ static inline void *phys_to_virt(phys_addr_t x)
 }

This enables tagged pointers for kernel addresses, and uses it to tag statically allocated kernel objects. This allows us to use a separate translation regime for kernel text in the next patch.

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm64/include/asm/memory.h        | 20 +++++++++++++++++++-
 arch/arm64/include/asm/pgtable-hwdef.h |  1 +
 arch/arm64/kernel/vmlinux.lds.S        |  4 ++--
 arch/arm64/mm/mmu.c                    |  4 ++--
 arch/arm64/mm/proc.S                   |  3 ++-
 5 files changed, 26 insertions(+), 6 deletions(-)

arch/arm64/include/asm/memory.h +11 -0

index f800d45ea226..7dfe1b0c9c01 100644
@@ -107,6 +107,10 @@
 #define MT_S2_NORMAL		0xf
 #define MT_S2_DEVICE_nGnRE	0x1
 
+#define __TEXT(x)	((x) & ~(UL(1) << 56))
+#define __VIRT(x)	((x) | (UL(1) << 56))
+#define __IS_TEXT(x)	(!((x) & (UL(1) << 56)))
+
 #ifndef __ASSEMBLY__
 
 extern phys_addr_t		memstart_addr;

@@ -141,9 +145,23 @@ static inline void *phys_to_virt(phys_addr_t x)
 }
 
 /*
+ * Return the physical address of a statically allocated object that
+ * is covered by the kernel Image mapping. We use tagged pointers to
+ * distinguish between the virtual linear and the virtual kimage range.
+ */
+static inline phys_addr_t __text_to_phys(unsigned long x)
+{
+	return __virt_to_phys(__VIRT(x));

arch/arm64/include/asm/memory.h +1 -0

+
 #define __va(x)			((void *)__phys_to_virt((phys_addr_t)(x)))
 #define pfn_to_kaddr(pfn)	__va((pfn) << PAGE_SHIFT)
 #define virt_to_pfn(x)      __phys_to_pfn(__virt_to_phys(x))

arch/arm64/include/asm/pgtable-hwdef.h +1 -0

index 5f930cc9ea83..8bcec4e626b4 100644
@@ -163,5 +163,6 @@
 #define TCR_TG1_64K		(UL(3) << 30)
 #define TCR_ASID16		(UL(1) << 36)
 #define TCR_TBI0		(UL(1) << 37)
+#define TCR_TBI1		(UL(1) << 38)
 
 #endif

arch/arm64/kernel/vmlinux.lds.S +2 -2

index 5d9d2dca530d..434ef407ef0f 100644
@@ -74,7 +74,7 @@ SECTIONS
 		*(.discard.*)
 	}
 
-	. = PAGE_OFFSET + TEXT_OFFSET;
+	. = __TEXT(PAGE_OFFSET) + TEXT_OFFSET;
 
 	.head.text : {
 		_text = .;

@@ -171,4 +171,4 @@ ASSERT(((__hyp_idmap_text_start + PAGE_SIZE) > __hyp_idmap_text_end),
 /*
  * If padding is applied before .head.text, virt<->phys conversions will fail.
  */
-ASSERT(_text == (PAGE_OFFSET + TEXT_OFFSET), "HEAD is misaligned")
+ASSERT(_text == (__TEXT(PAGE_OFFSET) + TEXT_OFFSET), "HEAD is misaligned")

arch/arm64/mm/mmu.c +2 -2

index c9267acb699c..43496748e3d9 100644
@@ -267,7 +267,7 @@ static void *late_alloc(unsigned long size)
 static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
 				  phys_addr_t size, pgprot_t prot)
 {
-	if (virt < VMALLOC_START) {
+	if (__VIRT(virt) < VMALLOC_START) {
 		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
 			&phys, virt);
 		return;

@@ -287,7 +287,7 @@ void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
 static void create_mapping_late(phys_addr_t phys, unsigned long virt,
 				  phys_addr_t size, pgprot_t prot)
 {
-	if (virt < VMALLOC_START) {
+	if (__VIRT(virt) < VMALLOC_START) {
 		pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
 			&phys, virt);
 		return;

arch/arm64/mm/proc.S +2 -1

index 28eebfb6af76..7f2d7f73bc93 100644
@@ -232,7 +232,8 @@ ENTRY(__cpu_setup)
 	 * both user and kernel.
 	 */
 	ldr	x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
-			TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
+			TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 | TCR_TBI1
+
 	/*
 	 * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in
 	 * TCR_EL1.

Signature · 1 line

1.8.3.2

This moves the fixmap translation tables to a dedicated region in the linker map. This is needed for a split kernel text from the linear mapping, to ensure that the contents of the tables rely on a single translation regime.

Also make the population of the translation levels conditional, so that the kernel text can share some levels of translation if needed.

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm64/include/asm/linkage.h |  2 ++
 arch/arm64/kernel/vmlinux.lds.S  | 15 ++++++++++-----
 arch/arm64/mm/mmu.c              | 15 +++++++++------
 3 files changed, 21 insertions(+), 11 deletions(-)

arch/arm64/include/asm/linkage.h +0 -0

index 636c1bced7d4..dc9354de6f52 100644

arch/arm64/include/asm/linkage.h +1 -0

+
 #endif

arch/arm64/kernel/vmlinux.lds.S +0 -4

index 434ef407ef0f..d3885043f0b7 100644
@@ -149,11 +149,16 @@ SECTIONS
 
 	BSS_SECTION(0, 0, 0)
 
-	. = ALIGN(PAGE_SIZE);
-	idmap_pg_dir = .;
-	. += IDMAP_DIR_SIZE;
-	swapper_pg_dir = .;

arch/arm64/kernel/vmlinux.lds.S +1 -0

+	}
 
 	_end = .;
 

arch/arm64/mm/mmu.c +0 -0

index 43496748e3d9..bb3ce41130f3 100644
@@ -549,12 +549,12 @@ void vmemmap_free(unsigned long start, unsigned long end)
 }
 #endif	/* CONFIG_SPARSEMEM_VMEMMAP */
 

arch/arm64/mm/mmu.c +0 -1

 static inline pud_t * fixmap_pud(unsigned long addr)

@@ -592,11 +592,14 @@ void __init early_fixmap_init(void)
 	unsigned long addr = FIXADDR_START;
 
 	pgd = pgd_offset_k(addr);
-	pgd_populate(&init_mm, pgd, bm_pud);

Hi Ard,

Quoted 4 lines · Ard Biesheuvel

This moves the fixmap translation tables to a dedicated region in the linker map. This is needed for a split kernel text from the linear mapping, to ensure that the contents of the tables rely on a single translation regime.

I'm not sure what you mean by this. Could you elaborate?

What problem would we have if we didn't have this section, and how does this section solve that?

Regardless, I have some comments on the implementation below.

Quoted 3 lines

Also make the population of the translation levels conditional, so that the kernel text can share some levels of translation if needed.

I guess you mean shared with the tables for the text mapping?

Please word this to be explicit w.r.t. what is shared between whom, and when.

Quoted 15 lines

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm64/include/asm/linkage.h |  2 ++
 arch/arm64/kernel/vmlinux.lds.S  | 15 ++++++++++-----
 arch/arm64/mm/mmu.c              | 15 +++++++++------
 3 files changed, 21 insertions(+), 11 deletions(-)

arch/arm64/include/asm/linkage.h +0 -0

index 636c1bced7d4..dc9354de6f52 100644
@@ -4,4 +4,6 @@
 #define __ALIGN              .align 4
 #define __ALIGN_STR  ".align 4"
 

arch/arm64/include/asm/linkage.h line 7

Quoted 1 line

arch/arm64/include/asm/linkage.h +1 -0

+#define __pgdir              __attribute__((__section__(".pgdir")))

It would be nice for this to also provide page alignment, like __page_aligned_bss that this replaces uses of.

Quoted 10 lines

arch/arm64/include/asm/linkage.h +1 -0

+
 #endif

arch/arm64/kernel/vmlinux.lds.S +0 -0

index 434ef407ef0f..d3885043f0b7 100644
@@ -149,11 +149,16 @@ SECTIONS
 
      BSS_SECTION(0, 0, 0)
 

Quoted 4 lines

arch/arm64/kernel/vmlinux.lds.S +0 -4

-     . = ALIGN(PAGE_SIZE);
-     idmap_pg_dir = .;
-     . += IDMAP_DIR_SIZE;
-     swapper_pg_dir = .;

arch/arm64/kernel/vmlinux.lds.S SECTIONS · line 160

Quoted 10 lines

arch/arm64/kernel/vmlinux.lds.S +9 -1

-     . += SWAPPER_DIR_SIZE;
+     .pgdir : {
+             . = ALIGN(PAGE_SIZE);
+             __pgdir_start = .;
+             idmap_pg_dir = .;
+             . += IDMAP_DIR_SIZE;
+             *(.pgdir)
+             swapper_pg_dir = .;
+             . += SWAPPER_DIR_SIZE;
+             __pgdir_stop = .;

Nit: __pgdir_end would be consitent with our other vmlinux.lds.S symbols.

Quoted 12 lines

arch/arm64/kernel/vmlinux.lds.S +1 -0

+     }
 
      _end = .;
 

arch/arm64/mm/mmu.c +0 -0

index 43496748e3d9..bb3ce41130f3 100644
@@ -549,12 +549,12 @@ void vmemmap_free(unsigned long start, unsigned long end)
 }
 #endif       /* CONFIG_SPARSEMEM_VMEMMAP */
 

arch/arm64/mm/mmu.c void vmemmap_free(unsigned long start, unsigned long end) · line 558

Quoted 10 lines

arch/arm64/mm/mmu.c +3 -3

-static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
+static pte_t bm_pte[PTRS_PER_PTE] __pgdir;
 #if CONFIG_ARM64_PGTABLE_LEVELS > 2
-static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss;
+static pmd_t bm_pmd[PTRS_PER_PMD] __pgdir;
 #endif
 #if CONFIG_ARM64_PGTABLE_LEVELS > 3
-static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss;
+static pud_t bm_pud[PTRS_PER_PUD] __pgdir;
 #endif

These used to be zeroed (by the bss init code in head.S), but now they won't be, and as they live after all the initialized data they could contain garbage if we're unlucky. So I suspect this is broken.

If we zero these before use then that's fine. We could define the swapper and idmap similarly in this C file for consistency (we zero those in head.S). That would bring all the page table allocations into the same file, and .pgdir could be simpler:

        .pgdir {
                . = ALIGN(PAGE_SIZE);
                __pgdir_start = .;
                *(.pgdir)
                __pgdir_end = .;
        }

Quoted 5 lines

arch/arm64/mm/mmu.c +0 -0

 static inline pud_t * fixmap_pud(unsigned long addr)

@@ -592,11 +592,14 @@ void __init early_fixmap_init(void)
      unsigned long addr = FIXADDR_START;
 

arch/arm64/mm/mmu.c void __init early_fixmap_init(void) · line 602

Quoted 12 lines

arch/arm64/mm/mmu.c +6 -3

      pgd = pgd_offset_k(addr);
-     pgd_populate(&init_mm, pgd, bm_pud);
+     if (pgd_none(*pgd))
+             pgd_populate(&init_mm, pgd, bm_pud);
      pud = pud_offset(pgd, addr);
-     pud_populate(&init_mm, pud, bm_pmd);
+     if (pud_none(*pud))
+             pud_populate(&init_mm, pud, bm_pmd);
      pmd = pmd_offset(pud, addr);
-     pmd_populate_kernel(&init_mm, pmd, bm_pte);
+     if (pmd_none(*pmd))
+             pmd_populate_kernel(&init_mm, pmd, bm_pte);

It's a bit confusing to have this change in the same patch as the other changes, given that this shouldn't be required yet. It might make more sense for this to live in the patch moving the text mapping.

Quoted 10 lines · Mark Rutland

Hi Ard,

Quoted 4 lines · Ard Biesheuvel

This moves the fixmap translation tables to a dedicated region in the linker map. This is needed for a split kernel text from the linear mapping, to ensure that the contents of the tables rely on a single translation regime.

I'm not sure what you mean by this. Could you elaborate?

Quoted 3 lines

What problem would we have if we didn't have this section, and how does this section solve that?

The pgd manipulation code is riddled with va/pa and pa/va translations, and uses both statically and dynamically allocated pages. Untangling all of that is not so easy, and it is simpler just to refer to those regions through the linear mapping exclusively.

Quoted 8 lines

Regardless, I have some comments on the implementation below.

Quoted 3 lines

Also make the population of the translation levels conditional, so that the kernel text can share some levels of translation if needed.

I guess you mean shared with the tables for the text mapping?

Quoted 2 lines

Please word this to be explicit w.r.t. what is shared between whom, and when.

Yes. Currently, the address space is split down the middle, so fixmap and linear always live in regions that are disjoint all the way up to different root pgdir entries. Once that changes, the fixmap code needs to be prepared for any of the levels it needs to populated having already been populated.

Quoted 15 lines

Quoted 15 lines

Signed-off-by Ard Biesheuvel <[email protected]>

---
 arch/arm64/include/asm/linkage.h |  2 ++
 arch/arm64/kernel/vmlinux.lds.S  | 15 ++++++++++-----
 arch/arm64/mm/mmu.c              | 15 +++++++++------
 3 files changed, 21 insertions(+), 11 deletions(-)

arch/arm64/include/asm/linkage.h +0 -0

index 636c1bced7d4..dc9354de6f52 100644
@@ -4,4 +4,6 @@
 #define __ALIGN              .align 4
 #define __ALIGN_STR  ".align 4"
 

Quoted 5 lines

arch/arm64/include/asm/linkage.h line 7

Quoted 1 line

arch/arm64/include/asm/linkage.h +1 -0

+#define __pgdir              __attribute__((__section__(".pgdir")))

It would be nice for this to also provide page alignment, like __page_aligned_bss that this replaces uses of.

I thought it wasn't necessary, because we allocate a full page for the highest level, but we should probably not rely on that.

Quoted 8 lines

Quoted 8 lines

arch/arm64/mm/mmu.c +0 -0

index 43496748e3d9..bb3ce41130f3 100644
@@ -549,12 +549,12 @@ void vmemmap_free(unsigned long start, unsigned long end)
 }
 #endif       /* CONFIG_SPARSEMEM_VMEMMAP */
 

Quoted 15 lines

arch/arm64/mm/mmu.c void vmemmap_free(unsigned long start, unsigned long end) · line 558

Quoted 10 lines

arch/arm64/mm/mmu.c +3 -3

-static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss;
+static pte_t bm_pte[PTRS_PER_PTE] __pgdir;
 #if CONFIG_ARM64_PGTABLE_LEVELS > 2
-static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss;
+static pmd_t bm_pmd[PTRS_PER_PMD] __pgdir;
 #endif
 #if CONFIG_ARM64_PGTABLE_LEVELS > 3
-static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss;
+static pud_t bm_pud[PTRS_PER_PUD] __pgdir;
 #endif

These used to be zeroed (by the bss init code in head.S), but now they won't be, and as they live after all the initialized data they could contain garbage if we're unlucky. So I suspect this is broken.

Actually, they are in this case, as the init code zeroes from the beginning of idmap to the end of swapper :-)
But that is something that should be more explicit I guess

I was wondering if we should worry about doing the zeroing with the caches off, which is not needed for fixmap

Quoted 11 lines

If we zero these before use then that's fine. We could define the swapper and idmap similarly in this C file for consistency (we zero those in head.S). That would bring all the page table allocations into the same file, and .pgdir could be simpler:

        .pgdir {
                . = ALIGN(PAGE_SIZE);
                __pgdir_start = .;
                *(.pgdir)
                __pgdir_end = .;
        }

This moves the virtual mapping of the kernel Image down into the lower half of the kernel virtual memory range, moving it out of the linear mapping.

This aligns PHYS_OFFSET down to an alignment that allows system RAM to be mapped using the largest blocks available, i.e., 1 GB blocks on 4 KB pages or 512 MB blocks on 64 KB pages.

Quoted 3 lines · Ard Biesheuvel

This aligns PHYS_OFFSET down to an alignment that allows system RAM to be mapped using the largest blocks available, i.e., 1 GB blocks on 4 KB pages or 512 MB blocks on 64 KB pages.

Does this assume that the platform RAM starts at such block aligned addresses? If not, is the unaligned range ignored?

arch/arm64/kernel/head.S +4 -2

index 16134608eecf..fd8434753372 100644
@@ -49,13 +49,15 @@
 #ifdef CONFIG_ARM64_64K_PAGES
 #define BLOCK_SHIFT	PAGE_SHIFT
 #define BLOCK_SIZE	PAGE_SIZE
-#define TABLE_SHIFT	PMD_SHIFT
 #else
 #define BLOCK_SHIFT	SECTION_SHIFT
 #define BLOCK_SIZE	SECTION_SIZE
-#define TABLE_SHIFT	PUD_SHIFT
 #endif
 
+#define TABLE_SHIFT	(BLOCK_SHIFT + PAGE_SHIFT - 3)
+#define TABLE_SIZE	(1 << TABLE_SHIFT)
+#define TABLE_MASK	(~(TABLE_SIZE - 1))
+
 #define KERNEL_START	_text
 #define KERNEL_END	_end
 

Quoted 25 lines · Ard Biesheuvel

arch/arm64/kernel/head.S +4 -2

index 16134608eecf..fd8434753372 100644
@@ -49,13 +49,15 @@
 #ifdef CONFIG_ARM64_64K_PAGES
 #define BLOCK_SHIFT  PAGE_SHIFT
 #define BLOCK_SIZE   PAGE_SIZE
-#define TABLE_SHIFT  PMD_SHIFT
 #else
 #define BLOCK_SHIFT  SECTION_SHIFT
 #define BLOCK_SIZE   SECTION_SIZE
-#define TABLE_SHIFT  PUD_SHIFT
 #endif
 
+#define TABLE_SHIFT  (BLOCK_SHIFT + PAGE_SHIFT - 3)
+#define TABLE_SIZE   (1 << TABLE_SHIFT)
+#define TABLE_MASK   (~(TABLE_SIZE - 1))
+
 #define KERNEL_START _text
 #define KERNEL_END   _end
 

@@ -237,7 +239,10 @@ ENTRY(stext)
      bl      el2_setup                       // Drop to EL1, w20=cpu_boot_mode
 

arch/arm64/kernel/head.S ENTRY(stext) · line 245

Quoted 6 lines

arch/arm64/kernel/head.S +4 -1

      adrp    x24, __PHYS_OFFSET
-     mov     x23, #KIMAGE_OFFSET
+     and     x23, x24, #~TABLE_MASK          // image offset
+     and     x24, x24, #TABLE_MASK           // PHYS_OFFSET
+     mov     x0, #KIMAGE_OFFSET
+     add     x23, x23, x0

I'm still trying to figure out how this works. Does the code imply that the kernel image can only be loaded within a block size of the PHYS_OFFSET? If that's the case, it's not too flexible.

My thoughts were to make PHYS_OFFSET permanently 0 (i.e. get rid of it entirely) or at least compute it at DT parsing time. I'm more inclined for making it 0 assuming that it doesn't break anything else (vmemmap virtual range may get slightly bigger but still not significant, basically max_phys_addr / sizeof(struct page)).

Quoted 26 lines · Catalin Marinas

Quoted 25 lines · Ard Biesheuvel

arch/arm64/kernel/head.S +4 -2

index 16134608eecf..fd8434753372 100644
@@ -49,13 +49,15 @@
 #ifdef CONFIG_ARM64_64K_PAGES
 #define BLOCK_SHIFT  PAGE_SHIFT
 #define BLOCK_SIZE   PAGE_SIZE
-#define TABLE_SHIFT  PMD_SHIFT
 #else
 #define BLOCK_SHIFT  SECTION_SHIFT
 #define BLOCK_SIZE   SECTION_SIZE
-#define TABLE_SHIFT  PUD_SHIFT
 #endif
 
+#define TABLE_SHIFT  (BLOCK_SHIFT + PAGE_SHIFT - 3)
+#define TABLE_SIZE   (1 << TABLE_SHIFT)
+#define TABLE_MASK   (~(TABLE_SIZE - 1))
+
 #define KERNEL_START _text
 #define KERNEL_END   _end
 

@@ -237,7 +239,10 @@ ENTRY(stext)
      bl      el2_setup                       // Drop to EL1, w20=cpu_boot_mode
 

Quoted 11 lines

arch/arm64/kernel/head.S ENTRY(stext) · line 245

Quoted 6 lines

arch/arm64/kernel/head.S +4 -1

      adrp    x24, __PHYS_OFFSET
-     mov     x23, #KIMAGE_OFFSET
+     and     x23, x24, #~TABLE_MASK          // image offset
+     and     x24, x24, #TABLE_MASK           // PHYS_OFFSET
+     mov     x0, #KIMAGE_OFFSET
+     add     x23, x23, x0

I'm still trying to figure out how this works. Does the code imply that the kernel image can only be loaded within a block size of the PHYS_OFFSET? If that's the case, it's not too flexible.

For now, yes.

Quoted 6 lines

My thoughts were to make PHYS_OFFSET permanently 0 (i.e. get rid of it entirely) or at least compute it at DT parsing time. I'm more inclined for making it 0 assuming that it doesn't break anything else (vmemmap virtual range may get slightly bigger but still not significant, basically max_phys_addr / sizeof(struct page)).

Making it zero would be an improvement, I suppose

Quoted 38 lines · Ard Biesheuvel

Quoted 36 lines · Catalin Marinas

Quoted 23 lines · Ard Biesheuvel

arch/arm64/kernel/head.S +4 -2

index 16134608eecf..fd8434753372 100644
@@ -49,13 +49,15 @@
 #ifdef CONFIG_ARM64_64K_PAGES
 #define BLOCK_SHIFT  PAGE_SHIFT
 #define BLOCK_SIZE   PAGE_SIZE
-#define TABLE_SHIFT  PMD_SHIFT
 #else
 #define BLOCK_SHIFT  SECTION_SHIFT
 #define BLOCK_SIZE   SECTION_SIZE
-#define TABLE_SHIFT  PUD_SHIFT
 #endif
 
+#define TABLE_SHIFT  (BLOCK_SHIFT + PAGE_SHIFT - 3)
+#define TABLE_SIZE   (1 << TABLE_SHIFT)
+#define TABLE_MASK   (~(TABLE_SIZE - 1))
+
 #define KERNEL_START _text
 #define KERNEL_END   _end
 

arch/arm64/kernel/head.S ENTRY(stext) · line 245

Quoted 8 lines

arch/arm64/kernel/head.S +4 -1

@@ -237,7 +239,10 @@ ENTRY(stext)
      bl      el2_setup                       // Drop to EL1, w20=cpu_boot_mode
 
      adrp    x24, __PHYS_OFFSET
-     mov     x23, #KIMAGE_OFFSET
+     and     x23, x24, #~TABLE_MASK          // image offset
+     and     x24, x24, #TABLE_MASK           // PHYS_OFFSET
+     mov     x0, #KIMAGE_OFFSET
+     add     x23, x23, x0

I'm still trying to figure out how this works. Does the code imply that the kernel image can only be loaded within a block size of the PHYS_OFFSET? If that's the case, it's not too flexible.

Quoted 1 line

For now, yes.

Quoted 6 lines

Quoted 5 lines

My thoughts were to make PHYS_OFFSET permanently 0 (i.e. get rid of it entirely) or at least compute it at DT parsing time. I'm more inclined for making it 0 assuming that it doesn't break anything else (vmemmap virtual range may get slightly bigger but still not significant, basically max_phys_addr / sizeof(struct page)).

Quoted 1 line

Making it zero would be an improvement, I suppose

Actually, wouldn't that reintroduce a similar VA range problem to the one I fixed the other day?

On Seattle, with its DRAM at 0x80_0000_0000, you wouldn't have enough space after PAGE_OFFSET

Quoted 39 lines · Ard Biesheuvel

Quoted 38 lines · Ard Biesheuvel

Quoted 36 lines · Catalin Marinas

Quoted 23 lines · Ard Biesheuvel

arch/arm64/kernel/head.S +4 -2

index 16134608eecf..fd8434753372 100644
@@ -49,13 +49,15 @@
 #ifdef CONFIG_ARM64_64K_PAGES
 #define BLOCK_SHIFT  PAGE_SHIFT
 #define BLOCK_SIZE   PAGE_SIZE
-#define TABLE_SHIFT  PMD_SHIFT
 #else
 #define BLOCK_SHIFT  SECTION_SHIFT
 #define BLOCK_SIZE   SECTION_SIZE
-#define TABLE_SHIFT  PUD_SHIFT
 #endif
 
+#define TABLE_SHIFT  (BLOCK_SHIFT + PAGE_SHIFT - 3)
+#define TABLE_SIZE   (1 << TABLE_SHIFT)
+#define TABLE_MASK   (~(TABLE_SIZE - 1))
+
 #define KERNEL_START _text
 #define KERNEL_END   _end
 

arch/arm64/kernel/head.S ENTRY(stext) · line 245

Quoted 8 lines

arch/arm64/kernel/head.S +4 -1

@@ -237,7 +239,10 @@ ENTRY(stext)
      bl      el2_setup                       // Drop to EL1, w20=cpu_boot_mode
 
      adrp    x24, __PHYS_OFFSET
-     mov     x23, #KIMAGE_OFFSET
+     and     x23, x24, #~TABLE_MASK          // image offset
+     and     x24, x24, #TABLE_MASK           // PHYS_OFFSET
+     mov     x0, #KIMAGE_OFFSET
+     add     x23, x23, x0

I'm still trying to figure out how this works. Does the code imply that the kernel image can only be loaded within a block size of the PHYS_OFFSET? If that's the case, it's not too flexible.

Quoted 1 line

Quoted 1 line

For now, yes.

Can we defer the setting of PHYS_OFFSET until we parse the DT memory nodes?

Quoted 6 lines

Quoted 6 lines

Quoted 5 lines

My thoughts were to make PHYS_OFFSET permanently 0 (i.e. get rid of it entirely) or at least compute it at DT parsing time. I'm more inclined for making it 0 assuming that it doesn't break anything else (vmemmap virtual range may get slightly bigger but still not significant, basically max_phys_addr / sizeof(struct page)).

Quoted 7 lines

Quoted 1 line

Making it zero would be an improvement, I suppose

Actually, wouldn't that reintroduce a similar VA range problem to the one I fixed the other day?

On Seattle, with its DRAM at 0x80_0000_0000, you wouldn't have enough space after PAGE_OFFSET

Ah, yes. When I thought about this PHYS_OFFSET == 0 in the past, we didn't have any patches and the VA range had to be sufficiently large.

--
Catalin

Quoted 40 lines · Catalin Marinas

Quoted 39 lines · Ard Biesheuvel

Quoted 38 lines · Ard Biesheuvel

Quoted 36 lines · Catalin Marinas

Quoted 23 lines · Ard Biesheuvel

arch/arm64/kernel/head.S +4 -2

index 16134608eecf..fd8434753372 100644
@@ -49,13 +49,15 @@
 #ifdef CONFIG_ARM64_64K_PAGES
 #define BLOCK_SHIFT  PAGE_SHIFT
 #define BLOCK_SIZE   PAGE_SIZE
-#define TABLE_SHIFT  PMD_SHIFT
 #else
 #define BLOCK_SHIFT  SECTION_SHIFT
 #define BLOCK_SIZE   SECTION_SIZE
-#define TABLE_SHIFT  PUD_SHIFT
 #endif
 
+#define TABLE_SHIFT  (BLOCK_SHIFT + PAGE_SHIFT - 3)
+#define TABLE_SIZE   (1 << TABLE_SHIFT)
+#define TABLE_MASK   (~(TABLE_SIZE - 1))
+
 #define KERNEL_START _text
 #define KERNEL_END   _end
 

arch/arm64/kernel/head.S ENTRY(stext) · line 245

Quoted 8 lines

arch/arm64/kernel/head.S +4 -1

@@ -237,7 +239,10 @@ ENTRY(stext)
      bl      el2_setup                       // Drop to EL1, w20=cpu_boot_mode
 
      adrp    x24, __PHYS_OFFSET
-     mov     x23, #KIMAGE_OFFSET
+     and     x23, x24, #~TABLE_MASK          // image offset
+     and     x24, x24, #TABLE_MASK           // PHYS_OFFSET
+     mov     x0, #KIMAGE_OFFSET
+     add     x23, x23, x0

I'm still trying to figure out how this works. Does the code imply that the kernel image can only be loaded within a block size of the PHYS_OFFSET? If that's the case, it's not too flexible.

Quoted 5 lines

Quoted 1 line

Quoted 1 line

For now, yes.

Can we defer the setting of PHYS_OFFSET until we parse the DT memory nodes?

I experimented a bit with that, but it is quite hairy. Any manipulation of the page tables goes through __va/__pa, so you need a valid PHYS_OFFSET there to ensure they point at the right physical region. But PHYS_OFFSET also needs to be small enough for the DT parsing code not to disregard regions that are below it. And then there is the memblock limit to ensure that early dynamically allocated page tables come from a region that is already mapped.

I think it may be doable, but it would require some significant hacking, e.g., call early_init_scan_dt() at its physical address with only the ID map loaded and the MMU and caches on, and only after that start populating the virtual address space. Or at least only populate the lower half, i.e., mappings below PAGE_OFFSET for the kernel and the FDT

Quoted 2 lines · Catalin Marinas

Can we defer the setting of PHYS_OFFSET until we parse the DT memory nodes?

I experimented a bit with that, but it is quite hairy. Any manipulation of the page tables goes through __va/__pa, so you need a valid PHYS_OFFSET there to ensure they point at the right physical region.

But PHYS_OFFSET also needs to be small enough for the DT parsing code not to disregard regions that are below it.

And then
there is the memblock limit to ensure that early dynamically allocated page tables come from a region that is already mapped.

I think it may be doable, but it would require some significant hacking, e.g., call early_init_scan_dt() at its physical address with only the ID map loaded and the MMU and caches on, and only after that start populating the virtual address space. Or at least only populate the lower half, i.e., mappings below PAGE_OFFSET for the kernel and the FDT

Quoted 8 lines · Ard Biesheuvel

Quoted 2 lines · Catalin Marinas

Can we defer the setting of PHYS_OFFSET until we parse the DT memory nodes?

I experimented a bit with that, but it is quite hairy. Any manipulation of the page tables goes through __va/__pa, so you need a valid PHYS_OFFSET there to ensure they point at the right physical region.

Yes, so we need set PHYS_OFFSET before we start using __va/__pa.

Quoted 2 lines

But PHYS_OFFSET also needs to be small enough for the DT parsing code not to disregard regions that are below it.

With PHYS_OFFSET as 0 initially, no regions would be dropped. But we could write a simplified early_init_dt_add_memory_arch() which avoids the PHYS_OFFSET check while setting the actual PHYS_OFFSET to the minimal address detected. I can see the generic function only uses __pa() to get the PHYS_OFFSET.

Quoted 3 lines

And then
there is the memblock limit to ensure that early dynamically allocated page tables come from a region that is already mapped.

By the time we start using memblock allocations, we have a PHYS_OFFSET set. The early DT parsing does not require any memory allocations AFAIK. We need to make sure that setup_machine_fdt creates a VA mapping of the DT and does not require the __va() macro (I thought I've seen some patches to use fixmap for DT).

Quoted 6 lines

I think it may be doable, but it would require some significant hacking, e.g., call early_init_scan_dt() at its physical address with only the ID map loaded and the MMU and caches on, and only after that start populating the virtual address space. Or at least only populate the lower half, i.e., mappings below PAGE_OFFSET for the kernel and the FDT

With some form of your patches, we already decouple the PAGE_OFFSET from the kernel text mapping. We map the latter at some very high KERNEL_PAGE_OFFSET, the DT via fixmap (which is part of the kernel data section, so mapped at the high KERNEL_PAGE_OFFSET). Once we start calling early_init_dt_add_memory_arch(), we set the real PAGE_OFFSET and are free to use __pa/__va after setup_machine_fdt(). The actual linear mappings will be created later via paging_init().

Quoted 12 lines · Catalin Marinas

Quoted 8 lines · Ard Biesheuvel

Quoted 2 lines · Catalin Marinas

Can we defer the setting of PHYS_OFFSET until we parse the DT memory nodes?

I experimented a bit with that, but it is quite hairy. Any manipulation of the page tables goes through __va/__pa, so you need a valid PHYS_OFFSET there to ensure they point at the right physical region.

Yes, so we need set PHYS_OFFSET before we start using __va/__pa.

Quoted 9 lines

Quoted 2 lines

But PHYS_OFFSET also needs to be small enough for the DT parsing code not to disregard regions that are below it.

With PHYS_OFFSET as 0 initially, no regions would be dropped. But we could write a simplified early_init_dt_add_memory_arch() which avoids the PHYS_OFFSET check while setting the actual PHYS_OFFSET to the minimal address detected. I can see the generic function only uses __pa() to get the PHYS_OFFSET.

Excellent idea.

Quoted 9 lines

Quoted 3 lines

And then
there is the memblock limit to ensure that early dynamically allocated page tables come from a region that is already mapped.

By the time we start using memblock allocations, we have a PHYS_OFFSET set. The early DT parsing does not require any memory allocations AFAIK. We need to make sure that setup_machine_fdt creates a VA mapping of the DT and does not require the __va() macro (I thought I've seen some patches to use fixmap for DT).

Yes, so did I :-)

But using early_fixmap() implies using the ordinary page tables manipulation code, which uses __pa/__va
So instead, I should refactor those patches to simply pick a VA offset and map the FDT there from head.S

Quoted 7 lines

Quoted 6 lines

I think it may be doable, but it would require some significant hacking, e.g., call early_init_scan_dt() at its physical address with only the ID map loaded and the MMU and caches on, and only after that start populating the virtual address space. Or at least only populate the lower half, i.e., mappings below PAGE_OFFSET for the kernel and the FDT

Quoted 7 lines

With some form of your patches, we already decouple the PAGE_OFFSET from the kernel text mapping. We map the latter at some very high KERNEL_PAGE_OFFSET, the DT via fixmap (which is part of the kernel data section, so mapped at the high KERNEL_PAGE_OFFSET). Once we start calling early_init_dt_add_memory_arch(), we set the real PAGE_OFFSET and are free to use __pa/__va after setup_machine_fdt(). The actual linear mappings will be created later via paging_init().

OK, that sounds like it could work.

The only thing to note is (and this should answer Mark's question in the other thread) is that the pgdir region should be mapped via the linear mapping as well.

The alternative is to make __va() check whether the input physical address falls within the pgdir region, and subtract the PAGE_OFFSET - KERNEL_PAGE_OFFSET from the return value, but this is less trivial than the __pa() counterpart which only needs to check a single bit.

So I propose we take defines KERNEL_PAGE_OFFSET as (PAGE_OFFSET - SZ_64M) and FDT_PAGE_OFFSET (or whatever you want to call it) as (PAGE_OFFSET - SZ_2M).
That way, the kernel, fdt and module space always share the same 128 MB window, which we can move around in the vmalloc space later if we want to implement kASLR.

Quoted 4 lines · Ard Biesheuvel

But using early_fixmap() implies using the ordinary page tables manipulation code, which uses __pa/__va
So instead, I should refactor those patches to simply pick a VA offset and map the FDT there from head.S

I haven't dug out those patches yet but in principle you should not care about __va, just __pa for populating the pmd/pud/pgd. Since with fixmap we pre-allocate the page tables in the kernel data section (bm_pud/pmd/pte), a __pa implementation that takes KERNEL_PAGE_OFFSET into account (as per these patches) is enough, you don't really care about the linear PAGE_OFFSET at this stage since you would not provide __pa with such virtual address until after setup_machine_fdt().

Quoted 6 lines · Catalin Marinas

Quoted 4 lines · Ard Biesheuvel

But using early_fixmap() implies using the ordinary page tables manipulation code, which uses __pa/__va
So instead, I should refactor those patches to simply pick a VA offset and map the FDT there from head.S

Quoted 7 lines

I haven't dug out those patches yet but in principle you should not care about __va, just __pa for populating the pmd/pud/pgd. Since with fixmap we pre-allocate the page tables in the kernel data section (bm_pud/pmd/pte), a __pa implementation that takes KERNEL_PAGE_OFFSET into account (as per these patches) is enough, you don't really care about the linear PAGE_OFFSET at this stage since you would not provide __pa with such virtual address until after setup_machine_fdt().

Actually, that's wrong, early_fixmap_init() uses pud/pmd_offset which in turn use the pmd_offset etc. and pmd_page_vaddr. These would not be mapped in the linear mapping at this stage. create_mapping() also uses the *_offset() functions, so we have the same issue.

So I think we still need to calculate the default initial PHYS_OFFSET in head.S as we currently do, based on the kernel load address so that __va() returns a mapping in the initial KERNEL_PAGE_OFFSET range. During our own early_init_dt_add_memory_arch(), we calculate the real PHYS_OFFSET but don't set it yet. We should defer the setting of the real PHYS_OFFSET until we map the first block of memory via create_mapping(). We know that we can't cope with allocations before the first block anyway, so this approach may work.

This implements a split mapping of the kernel Image using the AArch64 tagged pointers feature. The kernel text is emitted with bit 56 cleared, which can be used by the virt to phys translation to choose one translation regime or another.

The purpose is to allow PHYS_OFFSET to start at an arbitrary offset below the kernel image, so that memory below the kernel Image can be used, allowing the Image to be loaded anywhere in physical memory.

This series moves the kernel text right below PAGE_OFFSET, next to the modules area. PHYS_OFFSET is chosen to be a suitable aligned boundary somewhere below the kernel Image (1 GB or 512 MB depending on chosen page size).

Output from a QEMU/EFI boot:

System RAM:
memory[0x0] [0x00000040000000-0x000000bfffffff], 0x80000000 bytes flags: 0x0

Kernel image:
reserved[0x0] [0x0000005f680000-0x0000005fe68fff], 0x7e9000 bytes flags: 0x0

Virtual kernel memory layout:

    vmalloc : 0xffffff8000000000 - 0xffffffbdbfff0000   (   246 GB)
    vmemmap : 0xffffffbdc0000000 - 0xffffffbfc0000000   (     8 GB maximum)
              0xffffffbdc1000000 - 0xffffffbdc3000000   (    32 MB actual)
    fixed   : 0xffffffbff69fd000 - 0xffffffbff6c00000   (  2060 KB)
    PCI I/O : 0xffffffbff6e00000 - 0xffffffbff7e00000   (    16 MB)
    modules : 0xffffffbff8000000 - 0xffffffbffc000000   (    64 MB)
    memory  : 0xffffffc000000000 - 0xffffffc080000000   (  2048 MB)
      .init : 0xfeffffbffc783000 - 0xfeffffbffc7c6000   (   268 KB)
      .text : 0xfeffffbffc080000 - 0xfeffffbffc782344   (  7177 KB)
      .data : 0xfeffffbffc7ca000 - 0xfeffffbffc830c00   (   411 KB)

Note that this time, no relocations were harmed in the making of these patches. With added relocation support, it should be possible to move the combined modules and kernel text anywhere in the vmalloc area (for kaslr)

There are probably some places where the cleared bit 56 in the virtual address may cause trouble. Known problem is the module loader, but there are surely others.

Ard Biesheuvel (4):
arm64: use tagged pointers to distinguish kernel text from the linear
mapping
arm64: fixmap: move translation tables to dedicated region
arm64: move kernel text below PAGE_OFFSET
arm64: align PHYS_OFFSET to block size

 arch/arm64/include/asm/linkage.h       |  2 ++
 arch/arm64/include/asm/memory.h        | 27 +++++++++++++++++--
 arch/arm64/include/asm/pgtable-hwdef.h |  1 +
 arch/arm64/kernel/head.S               | 48 +++++++++++++++++++++++++---------
 arch/arm64/kernel/vmlinux.lds.S        | 19 +++++++++-----
 arch/arm64/mm/mmu.c                    | 21 +++++++++------
 arch/arm64/mm/proc.S                   |  3 ++-
 7 files changed, 91 insertions(+), 30 deletions(-)

Signature · 1 line

1.8.3.2

Hi Ard,

Quoted 4 lines · Ard Biesheuvel

This implements a split mapping of the kernel Image using the AArch64 tagged pointers feature. The kernel text is emitted with bit 56 cleared, which can be used by the virt to phys translation to choose one translation regime or another.

This looks neat!

I definitely want to see the linear mapping decoupled from the text mapping, but I'm rather worried by the prospect of tagged pointers in the kernel (there are a lot of edge cases for those w.r.t. the preservations of the tag bits), and if possible I'd like to avoid their use. I think that we can achieve the same effect by reorganising the VA layout within the 56 bits we have to play with.

Nit: please s/translation regime/mapping/ (or some other wording to that effect) for these patches; the architectural definition of "translation regime" differs from what you're referring to here, and we should avoid overloading that terminology.

Quoted 58 lines

The purpose is to allow PHYS_OFFSET to start at an arbitrary offset below the kernel image, so that memory below the kernel Image can be used, allowing the Image to be loaded anywhere in physical memory.

This series moves the kernel text right below PAGE_OFFSET, next to the modules area. PHYS_OFFSET is chosen to be a suitable aligned boundary somewhere below the kernel Image (1 GB or 512 MB depending on chosen page size).

Output from a QEMU/EFI boot:

System RAM:
memory[0x0] [0x00000040000000-0x000000bfffffff], 0x80000000 bytes flags: 0x0

Kernel image:
reserved[0x0] [0x0000005f680000-0x0000005fe68fff], 0x7e9000 bytes flags: 0x0

Virtual kernel memory layout:

    vmalloc : 0xffffff8000000000 - 0xffffffbdbfff0000   (   246 GB)
    vmemmap : 0xffffffbdc0000000 - 0xffffffbfc0000000   (     8 GB maximum)
              0xffffffbdc1000000 - 0xffffffbdc3000000   (    32 MB actual)
    fixed   : 0xffffffbff69fd000 - 0xffffffbff6c00000   (  2060 KB)
    PCI I/O : 0xffffffbff6e00000 - 0xffffffbff7e00000   (    16 MB)
    modules : 0xffffffbff8000000 - 0xffffffbffc000000   (    64 MB)
    memory  : 0xffffffc000000000 - 0xffffffc080000000   (  2048 MB)
      .init : 0xfeffffbffc783000 - 0xfeffffbffc7c6000   (   268 KB)
      .text : 0xfeffffbffc080000 - 0xfeffffbffc782344   (  7177 KB)
      .data : 0xfeffffbffc7ca000 - 0xfeffffbffc830c00   (   411 KB)

Note that this time, no relocations were harmed in the making of these patches. With added relocation support, it should be possible to move the combined modules and kernel text anywhere in the vmalloc area (for kaslr)

There are probably some places where the cleared bit 56 in the virtual address may cause trouble. Known problem is the module loader, but there are surely others.

Ard Biesheuvel (4):
arm64: use tagged pointers to distinguish kernel text from the linear
mapping
arm64: fixmap: move translation tables to dedicated region
arm64: move kernel text below PAGE_OFFSET
arm64: align PHYS_OFFSET to block size

 arch/arm64/include/asm/linkage.h       |  2 ++
 arch/arm64/include/asm/memory.h        | 27 +++++++++++++++++--
 arch/arm64/include/asm/pgtable-hwdef.h |  1 +
 arch/arm64/kernel/head.S               | 48 +++++++++++++++++++++++++---------
 arch/arm64/kernel/vmlinux.lds.S        | 19 +++++++++-----
 arch/arm64/mm/mmu.c                    | 21 +++++++++------
 arch/arm64/mm/proc.S                   |  3 ++-
 7 files changed, 91 insertions(+), 30 deletions(-)

--
1.8.3.2