riscv: Introduce CONFIG_RELOCATABLE
Alexandre Ghiti
reviewerPATCH 1/1 riscv: Introduce CONFIG_RELOCATABLE
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)
arch/riscv/Kconfig +11 -0
@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"
arch/riscv/Makefile +4 -1
@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endif
arch/riscv/boot/loader.lds.S +1 -1
@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)
arch/riscv/include/asm/page.h +3 -2
@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__
@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_load_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;
arch/riscv/kernel/head.S +2 -1
@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1
arch/riscv/kernel/vmlinux.lds.S +6 -4
@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>
@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;
arch/riscv/mm/Makefile +4 -0
@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endif
arch/riscv/mm/init.c +79 -13
159 lines not on this page · raw message
Signature · 1 line
2.20.1
Quoted 31 lines · Alexandre Ghiti
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)arch/riscv/Kconfig +1
-0@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE
arch/riscv/Kconfig config PGTABLE_LEVELS · line 176
Quoted 10 lines
arch/riscv/Kconfig +10
-0+ bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. +
I am sure this cannot be used for NOMMU support because of the page table mappings.
I guess this should depend on MMU.
Quoted 24 lines
arch/riscv/Kconfig +0
-0source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +0
-0
arch/riscv/boot/loader.lds.S ENTRY(_start) · line 13
Quoted 8 lines
arch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)
The loader.lds.S is used by NOMMU support so this should be separate patch.
Quoted 17 lines
arch/riscv/include/asm/page.h +2
-2@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__
arch/riscv/include/asm/page.h extern unsigned long pfn_base; · line 100
Quoted 4 lines
arch/riscv/include/asm/page.h +1
-0@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_load_addr;
Having this variable named kernel_virt_addr seems more logical to me.
Is it mandatory to use kernel_load_addr name ?
Quoted 234 lines
arch/riscv/include/asm/page.h +0
-0extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +6
-4@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +79
-14@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_load_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_load_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_load_addr);/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_load_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_load_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated +* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ; + +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_load_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_load_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_load_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_load_addr + load_sz; +for (va = kernel_load_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_load_addr), map_size,PAGE_KERNEL_EXEC );/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa (kernel_load_addr) && +__pa (kernel_load_addr) < end) + start =__pa (kernel_load_addr); map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) { --
Quoted 2 lines
2.20.1
Apart from minor comments above, I will certainly try this patch my end.
Regards,
Anup
Alex Ghiti
reviewerHi Anup,
Quoted 44 lines · Anup Patel
Quoted 30 lines · Alexandre Ghiti
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)arch/riscv/Kconfig +1
-0@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLEarch/riscv/Kconfig config PGTABLE_LEVELS · line 176
Quoted 10 lines
arch/riscv/Kconfig +10
-0+ bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. +I am sure this cannot be used for NOMMU support because of the page table mappings.
Quoted 1 line
I guess this should depend on MMU.
Yes you're right, I'll add a dependency on MMU in v2, thanks.
Quoted 26 lines
Quoted 26 lines
arch/riscv/Kconfig +0
-0source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +0
-0@@ -7,7 +7,7 @@ ENTRY(_start)
Quoted 8 lines
arch/riscv/boot/loader.lds.S ENTRY(_start) · line 13
Quoted 7 lines
arch/riscv/boot/loader.lds.S +1
-1SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)The
loader.lds.Sis used by NOMMU support so this should be separate patch.
This patch defines PAGE_OFFSET as a variable which will make NOMMU support not compilable, I'm not sure I should move that out of this patch, are you ?
Quoted 21 lines
Quoted 21 lines
arch/riscv/include/asm/page.h +2
-2@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */
Quoted 5 lines
arch/riscv/include/asm/page.h extern unsigned long pfn_base; · line 100
Quoted 1 line
arch/riscv/include/asm/page.h +1
-0+extern unsigned long kernel_load_addr;Having this variable named
kernel_virt_addrseems more logical to me.Is it mandatory to use
kernel_load_addrname ?
Not mandatory at all, I decided the naming. I agree with your suggestion, "load" refers better to physical addressing. I'll change to kernel_virt_addr in v2, thanks.
Quoted 232 lines
Quoted 232 lines
arch/riscv/include/asm/page.h +0
-0extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +6
-4@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +79
-13@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_load_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_load_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_load_addr);/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_load_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_load_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated +* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ; + +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_load_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_load_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_load_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_load_addr + load_sz; +for (va = kernel_load_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_load_addr), map_size,PAGE_KERNEL_EXEC );/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa (kernel_load_addr) && +__pa (kernel_load_addr) < end) + start =__pa (kernel_load_addr);
Quoted 7 lines
Quoted 5 lines
arch/riscv/mm/init.c +0
-1map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) { --2.20.1
Apart from minor comments above, I will certainly try this patch my end.
Great, more testing is always welcome, thanks.
Quoted 3 lines
Regards,
Anup
Alex
Quoted 315 lines · Alexandre Ghiti
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)arch/riscv/Kconfig +11
-0@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)arch/riscv/include/asm/page.h +3
-2@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_load_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +6
-4@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +76
-10@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_load_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_load_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_load_addr);/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_load_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_load_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated +* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ; + +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_load_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_load_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_load_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_load_addr + load_sz; +for (va = kernel_load_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_load_addr), map_size,PAGE_KERNEL_EXEC );/* Create fixed mapping for early FDT parsing */
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 9 lines
arch/riscv/mm/init.c +3
-3@@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa (kernel_load_addr) && +__pa (kernel_load_addr) < end) + start =__pa (kernel_load_addr);
Here should use __pa_symbol() instead of __pa() for kernel symbols.
I'm working on KASLR on top of this patch, it's work to me.
Quoted 6 lines
arch/riscv/mm/init.c +0
-1map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) { --2.20.1
Quoted 329 lines · Zong Li
Quoted 315 lines · Alexandre Ghiti
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)arch/riscv/Kconfig +11
-0@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)arch/riscv/include/asm/page.h +3
-2@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_load_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +6
-4@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +76
-10@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_load_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_load_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_load_addr);/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_load_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_load_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated +* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ; + +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_load_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_load_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_load_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_load_addr + load_sz; +for (va = kernel_load_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_load_addr), map_size,PAGE_KERNEL_EXEC );/* Create fixed mapping for early FDT parsing */ arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 9 lines
arch/riscv/mm/init.c +3
-3@@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa (kernel_load_addr) && +__pa (kernel_load_addr) < end) + start =__pa (kernel_load_addr);Here should use
__pa_symbol()instead of__pa()for kernel symbols.
Quoted 4 lines
I'm working on KASLR on top of this patch, it's work to me.
Reviewed-by Zong Li <[email protected]>Tested-by Zong Li <[email protected]>
Thanks for testing. I will not test since you already tested it.
Regards,
Anup
Quoted 7 lines
Quoted 7 lines
arch/riscv/mm/init.c +0
-1map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) { --2.20.1
Alex Ghiti
reviewerHi Zong,
Quoted 313 lines · Zong Li
Quoted 312 lines · Alexandre Ghiti
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)arch/riscv/Kconfig +11
-0@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)arch/riscv/include/asm/page.h +3
-2@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_load_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +6
-4@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +76
-10@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_load_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_load_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_load_addr);/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_load_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_load_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated +* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ; + +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_load_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_load_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_load_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_load_addr + load_sz; +for (va = kernel_load_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_load_addr), map_size,PAGE_KERNEL_EXEC );
Quoted 12 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 11 lines
arch/riscv/mm/init.c +3
-3/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa (kernel_load_addr) && +__pa (kernel_load_addr) < end) + start =__pa (kernel_load_addr);Here should use
__pa_symbol()instead of__pa()for kernel symbols.
I'll add it to the v2 too, thanks.
Quoted 1 line
I'm working on KASLR on top of this patch, it's work to me.
If I can help, don't hesitate, even if it is simply for testing :)
Quoted 3 lines
Reviewed-by Zong Li <[email protected]>Tested-by Zong Li <[email protected]>
Thanks for that,
Alex
Quoted 6 lines
Quoted 6 lines
arch/riscv/mm/init.c +0
-1map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) { --2.20.1
Quoted 341 lines · Alex Ghiti
Hi Zong,
Quoted 313 lines · Zong Li
Quoted 312 lines · Alexandre Ghiti
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)arch/riscv/Kconfig +11
-0@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)arch/riscv/include/asm/page.h +3
-2@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_load_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +6
-4@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +76
-10@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_load_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_load_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_load_addr);/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_load_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_load_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated +* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ; + +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_load_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_load_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_load_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_load_addr + load_sz; +for (va = kernel_load_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_load_addr), map_size,PAGE_KERNEL_EXEC );Quoted 12 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 11 lines
arch/riscv/mm/init.c +3
-3/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa (kernel_load_addr) && +__pa (kernel_load_addr) < end) + start =__pa (kernel_load_addr);Here should use
__pa_symbol()instead of__pa()for kernel symbols.I'll add it to the v2 too, thanks.
Quoted 1 line
I'm working on KASLR on top of this patch, it's work to me.
If I can help, don't hesitate, even if it is simply for testing :)
Quoted 1 line
Quoted 15 lines
Quoted 2 lines
Reviewed-by Zong Li <[email protected]>Tested-by Zong Li <[email protected]>Thanks for that,
Alex
Quoted 6 lines
Quoted 6 lines
arch/riscv/mm/init.c +0
-1map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) { --2.20.1
Hi all,
I find that there is a risk when building kernel as PIE. The problem is that hart_lottery will be put in .bss section rather than .sdata section, so the value of hart_lottery will be cleared by main hart. I post a patch to mark hart_lottery as sdata section by using gcc attribution.
Please refer to https://patchwork.kernel.org/patch/11364457/.
Feel free to give any feedbacks and comments. Thanks.
Quoted 361 lines · Zong Li
Quoted 341 lines · Alex Ghiti
Hi Zong,
Quoted 313 lines · Zong Li
Quoted 312 lines · Alexandre Ghiti
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>--- arch/riscv/Kconfig | 11 ++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 110 insertions(+), 22 deletions(-)arch/riscv/Kconfig +11
-0@@ -163,6 +163,17 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)arch/riscv/include/asm/page.h +3
-2@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_load_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_load_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -126,7 +126,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_load_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +6
-4@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -70,9 +70,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +76
-10@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_load_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_load_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_load_addr);/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_load_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_load_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated +* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ; + +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_load_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_load_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_load_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_load_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_load_addr + load_sz; +for (va = kernel_load_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_load_addr), map_size,PAGE_KERNEL_EXEC );Quoted 12 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 11 lines
arch/riscv/mm/init.c +3
-3/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa (kernel_load_addr) && +__pa (kernel_load_addr) < end) + start =__pa (kernel_load_addr);Here should use
__pa_symbol()instead of__pa()for kernel symbols.I'll add it to the v2 too, thanks.
Quoted 1 line
I'm working on KASLR on top of this patch, it's work to me.
If I can help, don't hesitate, even if it is simply for testing :)
Quoted 1 line
Quoted 15 lines
Quoted 2 lines
Reviewed-by Zong Li <[email protected]>Tested-by Zong Li <[email protected]>Thanks for that,
Alex
Quoted 6 lines
Quoted 6 lines
arch/riscv/mm/init.c +0
-1map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) { --2.20.1
Hi all,
Quoted 7 lines
I find that there is a risk when building kernel as PIE. The problem is that
hart_lotterywill be put in .bss section rather than .sdata section, so the value ofhart_lotterywill be cleared by main hart. I post a patch to markhart_lotteryas sdata section by using gcc attribution.Please refer to https://patchwork.kernel.org/patch/11364457/.
The CONFIG_RELOCATION has performance impact hence it is will not be enabled by default.
Also, we are moving away from fragile lottery based HART bring up to SBI HSM based HART bring up. I am sure by next year the lottery based HART bring-up will be obsolete and eventually we might even remove it altogether from Linux RISC-V kernel.
Regards,
Anup