riscv: Introduce CONFIG_RELOCATABLE
Alexandre Ghiti
reviewerPATCH RESEND v2 1/1 riscv: Introduce CONFIG_RELOCATABLE
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
---
Changes in v2:
- Make RELOCATABLE depend on MMU as suggested by Anup
- Rename kernel_load_addr into kernel_virt_addr as suggested by Anup
- Use __pa_symbol instead of __pa, as suggested by Zong
- Rebased on top of v5.6-rc3
- Tested with sv48 patchset
- Add Reviewed/Tested-by from Zong and Anup
arch/riscv/Kconfig | 12 +++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 111 insertions(+), 22 deletions(-)
arch/riscv/Kconfig +12 -0
@@ -163,6 +163,18 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + depends on MMU + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"
arch/riscv/Makefile +4 -1
@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endif
arch/riscv/boot/loader.lds.S +1 -1
@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)
arch/riscv/include/asm/page.h +3 -2
@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_virt_addr -#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))#ifndef __ASSEMBLY__
@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_virt_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;
arch/riscv/kernel/head.S +2 -1
@@ -131,7 +131,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_virt_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1
arch/riscv/kernel/vmlinux.lds.S +6 -4
@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>
@@ -71,9 +71,11 @@ SECTIONS EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + } _end = .;
arch/riscv/mm/Makefile +4 -0
@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endif
arch/riscv/mm/init.c +79 -13
159 lines not on this page · raw message
Signature · 1 line
2.20.1
Palmer Dabbelt
maintainerQuoted 89 lines · On Mon, 02 Mar 2020 21:44:37 PST (-0800), [email protected] wrote:
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>Reviewed-by Zong Li <[email protected]>Reviewed-by Anup Patel <[email protected]>Tested-by Zong Li <[email protected]>---
Changes in v2:
- Make RELOCATABLE depend on MMU as suggested by Anup
- Renamekernel_load_addrintokernel_virt_addras suggested by Anup
- Use__pa_symbolinstead of__pa, as suggested by Zong
- Rebased on top of v5.6-rc3
- Tested with sv48 patchset
- Add Reviewed/Tested-by from Zong and Anuparch/riscv/Kconfig | 12 +++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 111 insertions(+), 22 deletions(-)arch/riscv/Kconfig +12
-0@@ -163,6 +163,18 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + depends on MMU + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET; .payload : { *(.payload)arch/riscv/include/asm/page.h +0
-0
arch/riscv/include/asm/page.h line 34
Quoted 5 lines
arch/riscv/include/asm/page.h +1
-1@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_virt_addr
I assume we want to keep PAGE_OFFSET a constant for the non-relocatable systems. As it currently stands this is imposing a performance hit even when
arch/riscv/include/asm/page.h line 36
Quoted 2 lines
arch/riscv/include/asm/page.h +1
-1-#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))
This seems like it would cause issues if the kernel is relocated to high enough addresses that "kernel_virt_addr+KERN_VIRT_SIZE" overflows.
Quoted 39 lines
arch/riscv/include/asm/page.h +1
-0#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_virt_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -131,7 +131,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_virt_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +1
-1@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -71,9 +71,11 @@ SECTIONS
arch/riscv/kernel/vmlinux.lds.S SECTIONS · line 78
Quoted 10 lines
arch/riscv/kernel/vmlinux.lds.S +5
-3EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + }
It looks like the indentation is screwed up here: I see a mix of tabs/spaces that doesn't match the rest of the file.
Quoted 43 lines
arch/riscv/kernel/vmlinux.lds.S +0
-0_end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +6
-0@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_virt_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_virt_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };
arch/riscv/mm/init.c void __init setup_bootmem(void) · line 142
Quoted 6 lines
arch/riscv/mm/init.c +2
-1@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_virt_addr);
PAGE_OFFSET is kernel_virt_addr, so I don't see any reason to change these -- they account for a significant fraction of the diff.
Quoted 38 lines
arch/riscv/mm/init.c +24
-1/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_virt_addr) >>PGDIR_SHIFT ;BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_virt_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated
arch/riscv/mm/init.c static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) · line 412
Quoted 10 lines
arch/riscv/mm/init.c +10
-0+* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ;
This should at least provide a warning when it encounters an unresolvable relocation. Is it currently stands this just ignores all other runtime relocations, and while I can buy the argument there shouldn't be any (though I'd expect R_RISCV_{32,64} to show up?) we certainly shouldn't just silently skip them.
Quoted 93 lines
arch/riscv/mm/init.c +37
-11+ +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_virt_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_virt_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_virt_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_virt_addr + load_sz; +for (va = kernel_virt_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_virt_addr), map_size,PAGE_KERNEL_EXEC );/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa_symbol (kernel_virt_addr) && +__pa (kernel_virt_addr) < end) + start =__pa_symbol (kernel_virt_addr); map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) {
Alex Ghiti
reviewerHi Palmer,
Quoted 85 lines · Palmer Dabbelt
Quoted 84 lines · On Mon, 02 Mar 2020 21:44:37 PST (-0800), [email protected] wrote:
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>Reviewed-by Zong Li <[email protected]>Reviewed-by Anup Patel <[email protected]>Tested-by Zong Li <[email protected]>---
Changes in v2:
- Make RELOCATABLE depend on MMU as suggested by Anup
- Renamekernel_load_addrintokernel_virt_addras suggested by Anup
- Use__pa_symbolinstead of__pa, as suggested by Zong
- Rebased on top of v5.6-rc3
- Tested with sv48 patchset
- Add Reviewed/Tested-by from Zong and Anuparch/riscv/Kconfig | 12 +++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +-
arch/riscv/kernel/vmlinux.lds.S| 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++-----8 files changed, 111 insertions(+), 22 deletions(-)arch/riscv/Kconfig +0
-0@@ -163,6 +163,18 @@ config PGTABLE_LEVELSdefault 3 if 64BIT
default 2+config RELOCATABLE
+ bool
+ depends on MMU
+ help
+ This builds a kernel as a Position Independent Executable (PIE),
+ which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the
+ address it was linked at.
+ Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at.
+
source "arch/riscv/Kconfig.socs"menu "Platform type"
arch/riscv/Makefile +0
-0@@ -9,7 +9,10 @@ #OBJCOPYFLAGS := -O binary
arch/riscv/Makefile +4
-1-LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endififeq ($(CONFIG_DYNAMIC_FTRACE),y)
LDFLAGS_vmlinux:= --no-relax
endifarch/riscv/boot/loader.lds.S +0
-0@@ -7,7 +7,7 @@ ENTRY(_start)SECTIONS
{
- . =PAGE_OFFSET;
+ . =CONFIG_PAGE_OFFSET;
Quoted 15 lines
arch/riscv/include/asm/page.h line 34
Quoted 12 lines
.payload : {
*(.payload)arch/riscv/include/asm/page.h +1
-1@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_virt_addrI assume we want to keep
PAGE_OFFSETa constant for the non-relocatable systems. As it currently stands this is imposing a performance hit even when
Ok I can do that, thanks.
Quoted 5 lines
arch/riscv/include/asm/page.h line 36
Quoted 2 lines
arch/riscv/include/asm/page.h +1
-1-#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))This seems like it would cause issues if the kernel is relocated to high enough addresses that "kernel_virt_addr+KERN_VIRT_SIZE" overflows.
KERN_VIRT_SIZE is only used to define the size of VMALLOC zone and VMALLOC zone must aligned on 2MB (size of PMD page): so I simply used CONFIG_PAGE_OFFSET that guarantees that (since aligned on 1GB).
But I should define KERN_VIRT_SIZE by using kernel_virt_addr and make sure that VMALLOC zone is aligned on 2MB.
Ok I'll fix that in v3, thanks.
Quoted 42 lines
Quoted 42 lines
arch/riscv/include/asm/page.h +1
-0#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base; #define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT) #endif /* CONFIG_MMU */ +extern unsigned long kernel_virt_addr; extern unsigned long max_low_pfn; extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +0
-0@@ -131,7 +131,8 @@ clear_bss_done: #ifdef CONFIG_MMUrelocate:
/* Relocate return address */
- li a1,PAGE_OFFSET
+ la a1,kernel_virt_addr
+REG_La1, 0(a1)
la a2,_start
sub a1, a1, a2
add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +1
-1@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -71,9 +71,11 @@ SECTIONS EXCEPTION_TABLE(0x10)
Quoted 12 lines
arch/riscv/kernel/vmlinux.lds.S SECTIONS · line 78
Quoted 8 lines
arch/riscv/kernel/vmlinux.lds.S +5
-3- .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + }It looks like the indentation is screwed up here: I see a mix of tabs/spaces that doesn't match the rest of the file.
Thanks, I'll fix it.
Quoted 45 lines
Quoted 45 lines
arch/riscv/kernel/vmlinux.lds.S +0
-0_end = .;arch/riscv/mm/Makefile +0
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-onlyCFLAGS_init.o := -mcmodel=medany
arch/riscv/mm/Makefile +4
-0+ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif +ifdef
CONFIG_FTRACE
CFLAGS_REMOVE_init.o = -pg
endifarch/riscv/mm/init.c +6
-0@@ -12,6 +12,9 @@ #include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h>@@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page); extern char _start[]; void *dtb_early_va; +unsigned long kernel_virt_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_virt_addr); + static void __init zone_sizes_init(void) { unsigned long max_zone_pfns[MAX_NR_ZONES] = { 0, };@@ -132,7 +138,8 @@ void __init setup_bootmem(void) phys_addr_t end = reg->base + reg->size;
Quoted 7 lines
arch/riscv/mm/init.c void __init setup_bootmem(void) · line 142
Quoted 4 lines
arch/riscv/mm/init.c +2
-1if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size = min(reg->size, (phys_addr_t)-PAGE_OFFSET); + mem_size = min(reg->size, + (phys_addr_t)-kernel_virt_addr);
PAGE_OFFSETiskernel_virt_addr, so I don't see any reason to change these -- they account for a significant fraction of the diff.
You are totally right, those are leftovers from a previous version. Thanks for catching that, that greatly simplifies this patch.
Quoted 48 lines
Quoted 48 lines
arch/riscv/mm/init.c +32
-1/** Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va) if (mmu_enabled) return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); - pmd_num = (va - PAGE_OFFSET) >> PGDIR_SHIFT; + pmd_num = (va - kernel_virt_addr) >> PGDIR_SHIFT; BUG_ON(pmd_num >= NUM_EARLY_PMDS); return (uintptr_t)&early_pmd[pmd_num * PTRS_PER_PMD]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) #error "setup_vm() is called from head.S before relocate so it should not use absolute addressing." #endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ + Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; + uintptr_t link_addr = _AC(CONFIG_PAGE_OFFSET, UL); + /* + * This holds the offset between the linked virtual address and the + * relocated virtual address. + */ + uintptr_t reloc_offset = kernel_virt_addr - link_addr; + /* + * This holds the offset between linked virtual address and physical + * address whereas va_pa_offset holds the offset between relocated + * virtual address and physical address. + */ + uintptr_t va_link_pa_offset = link_addr - load_pa; + + for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { + Elf_Addr addr = (rela->r_offset - va_link_pa_offset); + Elf_Addr relocated_addr = rela->r_addend; +
Quoted 9 lines
arch/riscv/mm/init.c static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) · line 412
Quoted 2 lines
arch/riscv/mm/init.c +2
-0+ if (rela->r_info != R_RISCV_RELATIVE) + continue;This should at least provide a warning when it encounters an unresolvable relocation. Is it currently stands this just ignores all other runtime relocations, and while I can buy the argument there shouldn't be any (though I'd expect R_RISCV_{32,64} to show up?) we certainly shouldn't just silently skip them.
It's hard to warn that early in the boot process: what about we do like powerpc and warns at compile time ? In addition, it would be better that way since we can eliminate false-positive like in 43e76cd368fb (because the only R_RISCV_64 relocations are indeed false-positive).
Quoted 92 lines
Quoted 92 lines
arch/riscv/mm/init.c +15
-0+ + /* + * Make sure to not relocate vdso symbols like rt_sigreturn + * which are linked from the address 0 in vmlinux since + * vdso symbol addresses are actually used as an offset from + * mm->context.vdso in VDSO_OFFSET macro. + */ + if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {
uintptr_tva,end_va;arch/riscv/mm/init.c +5
-1@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) uintptr_t load_sz = (uintptr_t)(&_end) - load_pa; uintptr_t map_size = best_map_size(load_pa, MAX_EARLY_MAPPING_SIZE); - va_pa_offset = PAGE_OFFSET - load_pa; + va_pa_offset = kernel_virt_addr - load_pa; pfn_base = PFN_DOWN(load_pa); +#ifdef CONFIG_RELOCATABLE + /* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel
+ * cross over a 1G boundary, raise a bug since a part of the kernel + * would not get mapped.
+ */
+BUG_ON(SZ_1G - (kernel_virt_addr & (SZ_1G - 1)) < load_sz); +relocate_kernel(load_pa);arch/riscv/mm/init.c +2
-0+#endif +/*
* Enforce boot alignment requirements of RV32 and
* RV64 by only allowing PMD or PGD mappings.arch/riscv/mm/init.c +10
-10@@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) BUG_ON(map_size == PAGE_SIZE); /* Sanity check alignment and size */ - BUG_ON((PAGE_OFFSET % PGDIR_SIZE) != 0); + BUILD_BUG_ON((_AC(CONFIG_PAGE_OFFSET, UL) % PGDIR_SIZE) != 0); BUG_ON((load_pa % map_size) != 0); BUG_ON(load_sz > MAX_EARLY_MAPPING_SIZE);@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) create_pmd_mapping(fixmap_pmd, FIXADDR_START, (uintptr_t)fixmap_pte, PMD_SIZE, PAGE_TABLE); /* Setup trampoline PGD and PMD */ - create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, + create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, (uintptr_t)trampoline_pmd, PGDIR_SIZE, PAGE_TABLE); - create_pmd_mapping(trampoline_pmd, PAGE_OFFSET, + create_pmd_mapping(trampoline_pmd, kernel_virt_addr, load_pa, PMD_SIZE, PAGE_KERNEL_EXEC); #else /* Setup trampoline PGD */ - create_pgd_mapping(trampoline_pg_dir, PAGE_OFFSET, + create_pgd_mapping(trampoline_pg_dir, kernel_virt_addr, load_pa, PGDIR_SIZE, PAGE_KERNEL_EXEC); #endif@@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va = PAGE_OFFSET + load_sz; - for (va = PAGE_OFFSET; va < end_va; va += map_size) + end_va = kernel_virt_addr + load_sz; + for (va = kernel_virt_addr; va < end_va; va += map_size) create_pgd_mapping(early_pg_dir, va, - load_pa + (va - PAGE_OFFSET), + load_pa + (va - kernel_virt_addr), map_size, PAGE_KERNEL_EXEC); /* Create fixed mapping for early FDT parsing */@@ -457,9 +523,9 @@ static void __init setup_vm_final(void) break; if (memblock_is_nomap(reg)) continue; - if (start <= __pa(PAGE_OFFSET) && - __pa(PAGE_OFFSET) < end) - start = __pa(PAGE_OFFSET); + if (start <= __pa_symbol(kernel_virt_addr) && + __pa(kernel_virt_addr) < end) + start = __pa_symbol(kernel_virt_addr);
Quoted 2 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 531
Quoted 2 lines
arch/riscv/mm/init.c +0
-0map_size = best_map_size(start, end - start); for (pa = start; pa < end; pa += map_size) {
Thanks Palmer,
Alex
Quoted 84 lines · Palmer Dabbelt
Quoted 82 lines · On Mon, 02 Mar 2020 21:44:37 PST (-0800), [email protected] wrote:
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>Reviewed-by Zong Li <[email protected]>Reviewed-by Anup Patel <[email protected]>Tested-by Zong Li <[email protected]>---
Changes in v2:
- Make RELOCATABLE depend on MMU as suggested by Anup
- Renamekernel_load_addrintokernel_virt_addras suggested by Anup
- Use__pa_symbolinstead of__pa, as suggested by Zong
- Rebased on top of v5.6-rc3
- Tested with sv48 patchset
- Add Reviewed/Tested-by from Zong and Anuparch/riscv/Kconfig | 12 +++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 111 insertions(+), 22 deletions(-)arch/riscv/Kconfig +12
-0@@ -163,6 +163,18 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + depends on MMU + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET;
Quoted 16 lines
arch/riscv/include/asm/page.h line 34
Quoted 12 lines
arch/riscv/boot/loader.lds.S +0
-0.payload : { *(.payload)arch/riscv/include/asm/page.h +1
-1@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_virt_addrI assume we want to keep
PAGE_OFFSETa constant for the non-relocatable systems. As it currently stands this is imposing a performance hit even when
I had almost done the KASLR implementation on top of this patch. Actually, PAGE_OFFSET change is unnecessary in KASLR , because we would move kernel image to a random physical address as well, so the $pc will go to the relevant random virtual address. We need kernel_virt_addr to record the new destination, but keep PAGE_OFFSET to be CONFIG_PAGE_OFFSET is enough.
Quoted 6 lines
arch/riscv/include/asm/page.h line 36
Quoted 2 lines
arch/riscv/include/asm/page.h +1
-1-#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))This seems like it would cause issues if the kernel is relocated to high enough addresses that "kernel_virt_addr+KERN_VIRT_SIZE" overflows.
Based on the same reason, keep KERN_VIRT_SIZE to be -PAGE_OFFSET is good.
Quoted 98 lines
Quoted 39 lines
arch/riscv/include/asm/page.h +1
-0#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_virt_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -131,7 +131,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_virt_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +1
-1@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -71,9 +71,11 @@ SECTIONSarch/riscv/kernel/vmlinux.lds.S SECTIONS · line 78
Quoted 10 lines
arch/riscv/kernel/vmlinux.lds.S +5
-3EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + }It looks like the indentation is screwed up here: I see a mix of tabs/spaces that doesn't match the rest of the file.
Quoted 45 lines
arch/riscv/kernel/vmlinux.lds.S +0
-0_end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +6
-0@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_virt_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_virt_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;
Quoted 8 lines
arch/riscv/mm/init.c void __init setup_bootmem(void) · line 142
Quoted 4 lines
arch/riscv/mm/init.c +2
-1if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_virt_addr);
PAGE_OFFSETiskernel_virt_addr, so I don't see any reason to change these -- they account for a significant fraction of the diff.
kernel_virt_addr would be assigned to a random destination by KASLR, but here still should be PAGE_OFFSET rather than kernel_virt_addr as mentioned above.
Quoted 8 lines
arch/riscv/mm/init.c static phys_addr_t __init alloc_pmd(uintptr_t va) · line 279
Quoted 8 lines
arch/riscv/mm/init.c +1
-1/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_virt_addr) >>PGDIR_SHIFT ;
Here is the same, please use PAGE_OFFSET instead of kernel_virt_addr.
Quoted 126 lines
Quoted 30 lines
arch/riscv/mm/init.c +23
-0BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_virt_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated arch/riscv/mm/init.c static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) · line 412
Quoted 10 lines
arch/riscv/mm/init.c +10
-0+* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ;This should at least provide a warning when it encounters an unresolvable relocation. Is it currently stands this just ignores all other runtime relocations, and while I can buy the argument there shouldn't be any (though I'd expect R_RISCV_{32,64} to show up?) we certainly shouldn't just silently skip them.
Quoted 79 lines
arch/riscv/mm/init.c +34
-8+ +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_virt_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_virt_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_virt_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_virt_addr + load_sz; +for (va = kernel_virt_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_virt_addr), map_size,PAGE_KERNEL_EXEC );
Quoted 11 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 11 lines
arch/riscv/mm/init.c +3
-3/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa_symbol (kernel_virt_addr) && +__pa (kernel_virt_addr) < end) + start =__pa_symbol (kernel_virt_addr);
Here is the same, please use PAGE_OFFSET instead of kernel_virt_addr.
Quoted 3 lines
Quoted 3 lines
arch/riscv/mm/init.c +0
-0map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) {
Alex Ghiti
reviewerHi Zong,
Sorry for the response delay, please find below my comments.
Quoted 102 lines · Zong Li
Quoted 84 lines · Palmer Dabbelt
Quoted 82 lines · On Mon, 02 Mar 2020 21:44:37 PST (-0800), [email protected] wrote:
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>Reviewed-by Zong Li <[email protected]>Reviewed-by Anup Patel <[email protected]>Tested-by Zong Li <[email protected]>---
Changes in v2:
- Make RELOCATABLE depend on MMU as suggested by Anup
- Renamekernel_load_addrintokernel_virt_addras suggested by Anup
- Use__pa_symbolinstead of__pa, as suggested by Zong
- Rebased on top of v5.6-rc3
- Tested with sv48 patchset
- Add Reviewed/Tested-by from Zong and Anuparch/riscv/Kconfig | 12 +++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 111 insertions(+), 22 deletions(-)arch/riscv/Kconfig +12
-0@@ -163,6 +163,18 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + depends on MMU + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET;Quoted 16 lines
arch/riscv/include/asm/page.h line 34
Quoted 12 lines
arch/riscv/boot/loader.lds.S +0
-0.payload : { *(.payload)arch/riscv/include/asm/page.h +1
-1@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_virt_addrI assume we want to keep
PAGE_OFFSETa constant for the non-relocatable systems. As it currently stands this is imposing a performance hit even when
Quoted 4 lines
I had almost done the KASLR implementation on top of this patch. Actually,
PAGE_OFFSETchange is unnecessary in KASLR , because we would move kernel image to a random physical address as well, so the $pc will go to the relevant random virtual address. We need
I don't understand what you mean here, can you explain it a bit more ?
Quoted 2 lines
kernel_virt_addrto record the new destination, but keepPAGE_OFFSETto beCONFIG_PAGE_OFFSETis enough.
From my understanding, PAGE_OFFSET should represent the start of the direct mapping, so in case of a relocatable kernel, its value should reflect the offset too.
Is there any issue with having PAGE_OFFSET equal to kernel_virt_addr ?
If we use both, we will need to know precisely when we should use kernel_virt_addr or PAGE_OFFSET, which I think will be painful and error prone.
Quoted 258 lines
Quoted 7 lines
arch/riscv/include/asm/page.h line 36
Quoted 3 lines
arch/riscv/include/asm/page.h +1
-1-#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))This seems like it would cause issues if the kernel is relocated to high enough addresses that "kernel_virt_addr+KERN_VIRT_SIZE" overflows.
Based on the same reason, keep
KERN_VIRT_SIZEto be -PAGE_OFFSET is good.Quoted 98 lines
Quoted 39 lines
arch/riscv/include/asm/page.h +1
-0#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_virt_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -131,7 +131,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_virt_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +1
-1@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -71,9 +71,11 @@ SECTIONSarch/riscv/kernel/vmlinux.lds.S SECTIONS · line 78
Quoted 10 lines
arch/riscv/kernel/vmlinux.lds.S +5
-3EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + }It looks like the indentation is screwed up here: I see a mix of tabs/spaces that doesn't match the rest of the file.
Quoted 45 lines
arch/riscv/kernel/vmlinux.lds.S +0
-0_end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +6
-0@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_virt_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_virt_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;Quoted 8 lines
arch/riscv/mm/init.c void __init setup_bootmem(void) · line 142
Quoted 4 lines
arch/riscv/mm/init.c +2
-1if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_virt_addr);
PAGE_OFFSETiskernel_virt_addr, so I don't see any reason to change these -- they account for a significant fraction of the diff.
kernel_virt_addrwould be assigned to a random destination by KASLR, but here still should bePAGE_OFFSETrather thankernel_virt_addras mentioned above.Quoted 8 lines
arch/riscv/mm/init.c static phys_addr_t __init alloc_pmd(uintptr_t va) · line 279
Quoted 8 lines
arch/riscv/mm/init.c +1
-1/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_virt_addr) >>PGDIR_SHIFT ;Here is the same, please use
PAGE_OFFSETinstead ofkernel_virt_addr.Quoted 126 lines
Quoted 30 lines
arch/riscv/mm/init.c +23
-0BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_virt_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated arch/riscv/mm/init.c static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) · line 412
Quoted 10 lines
arch/riscv/mm/init.c +10
-0+* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ;This should at least provide a warning when it encounters an unresolvable relocation. Is it currently stands this just ignores all other runtime relocations, and while I can buy the argument there shouldn't be any (though I'd expect R_RISCV_{32,64} to show up?) we certainly shouldn't just silently skip them.
Quoted 79 lines
arch/riscv/mm/init.c +34
-8+ +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_virt_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_virt_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_virt_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_virt_addr + load_sz; +for (va = kernel_virt_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_virt_addr), map_size,PAGE_KERNEL_EXEC );
Quoted 17 lines
Quoted 11 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 11 lines
arch/riscv/mm/init.c +3
-3/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa_symbol (kernel_virt_addr) && +__pa (kernel_virt_addr) < end) + start =__pa_symbol (kernel_virt_addr);Here is the same, please use
PAGE_OFFSETinstead ofkernel_virt_addr.Quoted 3 lines
Quoted 3 lines
arch/riscv/mm/init.c +0
-0map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) {
Thanks,
Alex
Quoted 113 lines · Alex Ghiti
Hi Zong,
Sorry for the response delay, please find below my comments.
Quoted 102 lines · Zong Li
Quoted 84 lines · Palmer Dabbelt
Quoted 82 lines · On Mon, 02 Mar 2020 21:44:37 PST (-0800), [email protected] wrote:
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>Reviewed-by Zong Li <[email protected]>Reviewed-by Anup Patel <[email protected]>Tested-by Zong Li <[email protected]>---
Changes in v2:
- Make RELOCATABLE depend on MMU as suggested by Anup
- Renamekernel_load_addrintokernel_virt_addras suggested by Anup
- Use__pa_symbolinstead of__pa, as suggested by Zong
- Rebased on top of v5.6-rc3
- Tested with sv48 patchset
- Add Reviewed/Tested-by from Zong and Anuparch/riscv/Kconfig | 12 +++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 111 insertions(+), 22 deletions(-)arch/riscv/Kconfig +12
-0@@ -163,6 +163,18 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + depends on MMU + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET;Quoted 16 lines
arch/riscv/include/asm/page.h line 34
Quoted 12 lines
arch/riscv/boot/loader.lds.S +0
-0.payload : { *(.payload)arch/riscv/include/asm/page.h +1
-1@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_virt_addrI assume we want to keep
PAGE_OFFSETa constant for the non-relocatable systems. As it currently stands this is imposing a performance hit even whenQuoted 4 lines
I had almost done the KASLR implementation on top of this patch. Actually,
PAGE_OFFSETchange is unnecessary in KASLR , because we would move kernel image to a random physical address as well, so the $pc will go to the relevant random virtual address. We need
Quoted 9 lines
I don't understand what you mean here, can you explain it a bit more ?
Quoted 2 lines
kernel_virt_addrto record the new destination, but keepPAGE_OFFSETto beCONFIG_PAGE_OFFSETis enough.From my understanding,
PAGE_OFFSETshould represent the start of the
direct mapping, so in case of a relocatable kernel, its value should reflect the offset too.
Let me try to use the common RISC-V case to explain that.
In this v2 version, we could use a random virtual address to map the kernel image which is located in 0x8020_0000 physical address, then we could achieve the purpose of randomization, but it only happens on virtual address, the kernel image is still in 0x8020_0000 physical address, so we would also try to move the physical location of kernel image by a random offset.
For example, assume there is a 1G size DDR, the kernel image size is 10MB and we get a random kernel offset 0x100_0000, finally, kernel image would locate in 0x8120_0000, and run on corresponding virtual address 0xffff_ffe0_0100_0000 in CPU view. The direct mapping range is still got from 0x8020_0000 to 0xc000_0000, that means, there are available free pages from 0x8020_0000 to 0x811f_ffff, and a reserved region for kernel image from 0x8120_0000 to 0x81c0_0000, the rest of the memory are free pages as well (in fact, there are other reserved regions such as dtb, initrd and so on). Hence, we don't change PAGE_OFFSET, the PAGE_OFFSET still represent the start of the direct mapping as you mentioned, but the location of kernel image which in direct mapping would be unpredictable.
Quoted 1 line
Is there any issue with having
PAGE_OFFSETequal tokernel_virt_addr?
If we make PAGE_OFFSET be the new random kernel start address, the system would lose the part of memory after we also moving kernel image to a new physical location, that means, the memory which prior of new random kernel start address is disappeared, because some reasons like it only creates page table start from new physical address, the memblock only sees the memory from __pa(PAGE_OFFSET), and etc.
Quoted 4 lines
If we use both, we will need to know precisely when we should use
kernel_virt_addrorPAGE_OFFSET, which I think will be painful and error prone.
There is a use case, we could get kernel offset by the expression 'kernel_virt_addr - PAGE_OFFSET'.
It seems to me that I could post the RFC patches first, but it depends on other patch series, include your relocation implementation.
Quoted 280 lines
Quoted 259 lines
Quoted 8 lines
arch/riscv/include/asm/page.h line 36
Quoted 4 lines
arch/riscv/include/asm/page.h +1
-1-#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))This seems like it would cause issues if the kernel is relocated to high enough addresses that "kernel_virt_addr+KERN_VIRT_SIZE" overflows.
Based on the same reason, keep
KERN_VIRT_SIZEto be -PAGE_OFFSET is good.Quoted 98 lines
Quoted 39 lines
arch/riscv/include/asm/page.h +1
-0#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_virt_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -131,7 +131,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_virt_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +1
-1@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -71,9 +71,11 @@ SECTIONSarch/riscv/kernel/vmlinux.lds.S SECTIONS · line 78
Quoted 10 lines
arch/riscv/kernel/vmlinux.lds.S +5
-3EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + }It looks like the indentation is screwed up here: I see a mix of tabs/spaces that doesn't match the rest of the file.
Quoted 45 lines
arch/riscv/kernel/vmlinux.lds.S +0
-0_end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +6
-0@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_virt_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_virt_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;Quoted 8 lines
arch/riscv/mm/init.c void __init setup_bootmem(void) · line 142
Quoted 4 lines
arch/riscv/mm/init.c +2
-1if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_virt_addr);
PAGE_OFFSETiskernel_virt_addr, so I don't see any reason to change these -- they account for a significant fraction of the diff.
kernel_virt_addrwould be assigned to a random destination by KASLR, but here still should bePAGE_OFFSETrather thankernel_virt_addras mentioned above.Quoted 8 lines
arch/riscv/mm/init.c static phys_addr_t __init alloc_pmd(uintptr_t va) · line 279
Quoted 8 lines
arch/riscv/mm/init.c +1
-1/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_virt_addr) >>PGDIR_SHIFT ;Here is the same, please use
PAGE_OFFSETinstead ofkernel_virt_addr.Quoted 126 lines
Quoted 30 lines
arch/riscv/mm/init.c +23
-0BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_virt_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated arch/riscv/mm/init.c static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) · line 412
Quoted 10 lines
arch/riscv/mm/init.c +10
-0+* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ;This should at least provide a warning when it encounters an unresolvable relocation. Is it currently stands this just ignores all other runtime relocations, and while I can buy the argument there shouldn't be any (though I'd expect R_RISCV_{32,64} to show up?) we certainly shouldn't just silently skip them.
Quoted 79 lines
arch/riscv/mm/init.c +34
-8+ +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_virt_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_virt_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_virt_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_virt_addr + load_sz; +for (va = kernel_virt_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_virt_addr), map_size,PAGE_KERNEL_EXEC );Quoted 17 lines
Quoted 11 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 11 lines
arch/riscv/mm/init.c +3
-3/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa_symbol (kernel_virt_addr) && +__pa (kernel_virt_addr) < end) + start =__pa_symbol (kernel_virt_addr);Here is the same, please use
PAGE_OFFSETinstead ofkernel_virt_addr.Quoted 3 lines
Quoted 3 lines
arch/riscv/mm/init.c +0
-0map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) {Thanks,
Alex
Alex Ghiti
reviewerHi Zong,
Quoted 160 lines · Zong Li
Quoted 113 lines · Alex Ghiti
Hi Zong,
Sorry for the response delay, please find below my comments.
Quoted 102 lines · Zong Li
Quoted 84 lines · Palmer Dabbelt
Quoted 82 lines · On Mon, 02 Mar 2020 21:44:37 PST (-0800), [email protected] wrote:
This config allows to compile the kernel as PIE and to relocate it at any virtual address at runtime: this paves the way to KASLR and to 4-level page table folding at runtime. Runtime relocation is possible since relocation metadata are embedded into the kernel.
Note that relocating at runtime introduces an overhead even if the kernel is loaded at the same address it was linked at and that the compiler options are those used in arm64 which uses the same RELA relocation format.
Signed-off-by Alexandre Ghiti <[email protected]>Reviewed-by Zong Li <[email protected]>Reviewed-by Anup Patel <[email protected]>Tested-by Zong Li <[email protected]>---
Changes in v2:
- Make RELOCATABLE depend on MMU as suggested by Anup
- Renamekernel_load_addrintokernel_virt_addras suggested by Anup
- Use__pa_symbolinstead of__pa, as suggested by Zong
- Rebased on top of v5.6-rc3
- Tested with sv48 patchset
- Add Reviewed/Tested-by from Zong and Anuparch/riscv/Kconfig | 12 +++++ arch/riscv/Makefile | 5 +- arch/riscv/boot/loader.lds.S | 2 +- arch/riscv/include/asm/page.h | 5 +- arch/riscv/kernel/head.S | 3 +- arch/riscv/kernel/vmlinux.lds.S | 10 ++-- arch/riscv/mm/Makefile | 4 ++ arch/riscv/mm/init.c | 92 ++++++++++++++++++++++++++++----- 8 files changed, 111 insertions(+), 22 deletions(-)arch/riscv/Kconfig +12
-0@@ -163,6 +163,18 @@ config PGTABLE_LEVELS default 3 if 64BIT default 2 +config RELOCATABLE + bool + depends on MMU + help + This builds a kernel as a Position Independent Executable (PIE), + which retains all relocation metadata required to relocate the + kernel binary at runtime to a different virtual address than the + address it was linked at. + Since RISCV uses the RELA relocation format, this requires a + relocation pass at runtime even if the kernel is loaded at the + same address it was linked at. + source "arch/riscv/Kconfig.socs" menu "Platform type"arch/riscv/Makefile +4
-1@@ -9,7 +9,10 @@ # OBJCOPYFLAGS := -O binary -LDFLAGS_vmlinux := +ifeq ($(CONFIG_RELOCATABLE),y) +LDFLAGS_vmlinux := -shared -Bsymbolic -z notext -z norelro +KBUILD_CFLAGS += -fPIE +endif ifeq ($(CONFIG_DYNAMIC_FTRACE),y) LDFLAGS_vmlinux := --no-relax endifarch/riscv/boot/loader.lds.S +1
-1@@ -7,7 +7,7 @@ ENTRY(_start) SECTIONS { - . = PAGE_OFFSET; + . = CONFIG_PAGE_OFFSET;Quoted 16 lines
arch/riscv/include/asm/page.h line 34
Quoted 12 lines
arch/riscv/boot/loader.lds.S +0
-0.payload : { *(.payload)arch/riscv/include/asm/page.h +1
-1@@ -31,9 +31,9 @@* When not using MMU this corresponds to the first free page in * physical memory (aligned on a page boundary). */ -#define PAGE_OFFSET _AC(CONFIG_PAGE_OFFSET, UL) +#define PAGE_OFFSET kernel_virt_addrI assume we want to keep
PAGE_OFFSETa constant for the non-relocatable systems. As it currently stands this is imposing a performance hit even whenQuoted 4 lines
I had almost done the KASLR implementation on top of this patch. Actually,
PAGE_OFFSETchange is unnecessary in KASLR , because we would move kernel image to a random physical address as well, so the $pc will go to the relevant random virtual address. We needQuoted 9 lines
I don't understand what you mean here, can you explain it a bit more ?
Quoted 2 lines
kernel_virt_addrto record the new destination, but keepPAGE_OFFSETto beCONFIG_PAGE_OFFSETis enough.From my understanding,
PAGE_OFFSETshould represent the start of the
direct mapping, so in case of a relocatable kernel, its value should reflect the offset too.Let me try to use the common RISC-V case to explain that.
In this v2 version, we could use a random virtual address to map the kernel image which is located in 0x8020_0000 physical address, then we could achieve the purpose of randomization, but it only happens on virtual address, the kernel image is still in 0x8020_0000 physical address, so we would also try to move the physical location of kernel image by a random offset.
For example, assume there is a 1G size DDR, the kernel image size is 10MB and we get a random kernel offset 0x100_0000, finally, kernel image would locate in 0x8120_0000, and run on corresponding virtual address 0xffff_ffe0_0100_0000 in CPU view. The direct mapping range is still got from 0x8020_0000 to 0xc000_0000, that means, there are available free pages from 0x8020_0000 to 0x811f_ffff, and a reserved region for kernel image from 0x8120_0000 to 0x81c0_0000, the rest of the memory are free pages as well (in fact, there are other reserved regions such as dtb, initrd and so on). Hence, we don't change
PAGE_OFFSET, thePAGE_OFFSETstill represent the start of the direct mapping as you mentioned, but the location of kernel image which in direct mapping would be unpredictable.Quoted 1 line
Is there any issue with having
PAGE_OFFSETequal tokernel_virt_addr?If we make
PAGE_OFFSETbe the new random kernel start address, the system would lose the part of memory after we also moving kernel image to a new physical location, that means, the memory which prior of new random kernel start address is disappeared, because some reasons like it only creates page table start from new physical address, the memblock only sees the memory from__pa(PAGE_OFFSET), and etc.Quoted 4 lines
If we use both, we will need to know precisely when we should use
kernel_virt_addrorPAGE_OFFSET, which I think will be painful and error prone.
Quoted 6 lines
There is a use case, we could get kernel offset by the expression '
kernel_virt_addr-PAGE_OFFSET'.It seems to me that I could post the RFC patches first, but it depends on other patch series, include your relocation implementation.
You are totally right, thanks for your explanation. I'll fix that in v3 then.
I'll take a look at your KASLR implementation soon,
Thanks,
Alex
Quoted 280 lines
Quoted 280 lines
Quoted 259 lines
Quoted 8 lines
arch/riscv/include/asm/page.h line 36
Quoted 4 lines
arch/riscv/include/asm/page.h +1
-1-#define KERN_VIRT_SIZE (-PAGE_OFFSET) +#define KERN_VIRT_SIZE (-_AC(CONFIG_PAGE_OFFSET, UL))This seems like it would cause issues if the kernel is relocated to high enough addresses that "kernel_virt_addr+KERN_VIRT_SIZE" overflows.
Based on the same reason, keep
KERN_VIRT_SIZEto be -PAGE_OFFSET is good.Quoted 98 lines
Quoted 39 lines
arch/riscv/include/asm/page.h +1
-0#ifndef __ASSEMBLY__@@ -97,6 +97,7 @@ extern unsigned long pfn_base;#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)#endif /* CONFIG_MMU */ +extern unsigned long kernel_virt_addr;extern unsigned long max_low_pfn;extern unsigned long min_low_pfn;arch/riscv/kernel/head.S +2
-1@@ -131,7 +131,8 @@ clear_bss_done: #ifdef CONFIG_MMU relocate: /* Relocate return address */ - li a1, PAGE_OFFSET + la a1, kernel_virt_addr + REG_L a1, 0(a1) la a2, _start sub a1, a1, a2 add ra, ra, a1arch/riscv/kernel/vmlinux.lds.S +1
-1@@ -4,7 +4,7 @@ * Copyright (C) 2017 SiFive */ -#define LOAD_OFFSET PAGE_OFFSET +#define LOAD_OFFSET CONFIG_PAGE_OFFSET #include <asm/vmlinux.lds.h> #include <asm/page.h> #include <asm/cache.h>@@ -71,9 +71,11 @@ SECTIONSarch/riscv/kernel/vmlinux.lds.S SECTIONS · line 78
Quoted 10 lines
arch/riscv/kernel/vmlinux.lds.S +5
-3EXCEPTION_TABLE(0x10) - .rel.dyn : { - *(.rel.dyn*) - } + .rela.dyn : ALIGN(8) { + __rela_dyn_start = .; + *(.rela .rela*) + __rela_dyn_end = .; + }It looks like the indentation is screwed up here: I see a mix of tabs/spaces that doesn't match the rest of the file.
Quoted 45 lines
arch/riscv/kernel/vmlinux.lds.S +0
-0_end = .;arch/riscv/mm/Makefile +4
-0@@ -1,6 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS_init.o := -mcmodel=medany +ifdef CONFIG_RELOCATABLE +CFLAGS_init.o += -fno-pie +endif + ifdef CONFIG_FTRACE CFLAGS_REMOVE_init.o = -pg endifarch/riscv/mm/init.c +6
-0@@ -12,6 +12,9 @@#include <linux/sizes.h> #include <linux/of_fdt.h> #include <linux/libfdt.h> +#ifdef CONFIG_RELOCATABLE +#include <linux/elf.h> +#endif #include <asm/fixmap.h> #include <asm/tlbflush.h> @@ -28,6 +31,9 @@ EXPORT_SYMBOL(empty_zero_page);extern char _start[];void *dtb_early_va; +unsigned long kernel_virt_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +EXPORT_SYMBOL (kernel_virt_addr); +static void __init zone_sizes_init (void ) {unsigned long max_zone_pfns[MAX_NR_ZONES ] = {0 , };@@ -132,7 +138,8 @@ void __init setup_bootmem(void)phys_addr_t end = reg->base + reg->size;Quoted 8 lines
arch/riscv/mm/init.c void __init setup_bootmem(void) · line 142
Quoted 4 lines
arch/riscv/mm/init.c +2
-1if (reg->base <= vmlinux_end && vmlinux_end <= end) { - mem_size =min (reg->size, (phys_addr_t)-PAGE_OFFSET ); + mem_size =min (reg->size, + (phys_addr_t)-kernel_virt_addr);
PAGE_OFFSETiskernel_virt_addr, so I don't see any reason to change these -- they account for a significant fraction of the diff.
kernel_virt_addrwould be assigned to a random destination by KASLR, but here still should bePAGE_OFFSETrather thankernel_virt_addras mentioned above.Quoted 8 lines
arch/riscv/mm/init.c static phys_addr_t __init alloc_pmd(uintptr_t va) · line 279
Quoted 8 lines
arch/riscv/mm/init.c +1
-1/* * Remove memblock from the end of usable area to the @@ -269,7 +276,7 @@ static phys_addr_t __init alloc_pmd(uintptr_t va)if (mmu_enabled)return memblock_phys_alloc (PAGE_SIZE ,PAGE_SIZE ); - pmd_num = (va -PAGE_OFFSET ) >>PGDIR_SHIFT ; + pmd_num = (va - kernel_virt_addr) >>PGDIR_SHIFT ;Here is the same, please use
PAGE_OFFSETinstead ofkernel_virt_addr.Quoted 126 lines
Quoted 30 lines
arch/riscv/mm/init.c +23
-0BUG_ON (pmd_num >=NUM_EARLY_PMDS );return (uintptr_t )&early_pmd[pmd_num *PTRS_PER_PMD ]; }@@ -370,6 +377,54 @@ static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size)#error "setup_vm() is called from head.S before relocate so it should not use absolute addressing."#endif +#ifdef CONFIG_RELOCATABLE +extern unsigned long __rela_dyn_start, __rela_dyn_end; + +#ifdef CONFIG_64BIT +#define Elf_Rela Elf64_Rela +#define Elf_Addr Elf64_Addr +#else +#define Elf_Rela Elf32_Rela +#define Elf_Addr Elf32_Addr +#endif + +void __init relocate_kernel (uintptr_t load_pa) +{ +Elf_Rela *rela = (Elf_Rela *)&__rela_dyn_start; +uintptr_t link_addr =_AC (CONFIG_PAGE_OFFSET ,UL ); +/* +* This holds the offset between the linked virtual address and the +* relocated virtual address. +*/ +uintptr_t reloc_offset = kernel_virt_addr - link_addr; +/* +* This holds the offset between linked virtual address and physical +* address whereas va_pa_offset holds the offset between relocated arch/riscv/mm/init.c static uintptr_t __init best_map_size(phys_addr_t base, phys_addr_t size) · line 412
Quoted 10 lines
arch/riscv/mm/init.c +10
-0+* virtual address and physical address. +*/ +uintptr_t va_link_pa_offset = link_addr - load_pa; + +for ( ; rela < (Elf_Rela *)&__rela_dyn_end; rela++) { +Elf_Addr addr = (rela->r_offset - va_link_pa_offset); +Elf_Addr relocated_addr = rela->r_addend; + +if (rela->r_info !=R_RISCV_RELATIVE ) +continue ;This should at least provide a warning when it encounters an unresolvable relocation. Is it currently stands this just ignores all other runtime relocations, and while I can buy the argument there shouldn't be any (though I'd expect R_RISCV_{32,64} to show up?) we certainly shouldn't just silently skip them.
Quoted 79 lines
arch/riscv/mm/init.c +34
-8+ +/* +* Make sure to not relocate vdso symbols like rt_sigreturn +* which are linked from the address 0 in vmlinux since +* vdso symbol addresses are actually used as an offset from +* mm->context.vdso in VDSO_OFFSET macro. +*/ +if (relocated_addr >= link_addr) + relocated_addr += reloc_offset; + + *(Elf_Addr *)addr = relocated_addr; + } +} +#endif + asmlinkage void __init setup_vm(uintptr_t dtb_pa) {uintptr_t va, end_va;@@ -377,9 +432,20 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)uintptr_t load_sz = (uintptr_t )(&_end) - load_pa;uintptr_t map_size =best_map_size (load_pa,MAX_EARLY_MAPPING_SIZE ); - va_pa_offset =PAGE_OFFSET - load_pa; + va_pa_offset = kernel_virt_addr - load_pa; pfn_base =PFN_DOWN (load_pa); +#ifdef CONFIG_RELOCATABLE +/* +* Early page table uses only one PGDIR, which makes it possible +* to map 1GB aligned on 1GB: if the relocation offset makes the kernel +* cross over a 1G boundary, raise a bug since a part of the kernel +* would not get mapped. +*/ +BUG_ON (SZ_1G - (kernel_virt_addr & (SZ_1G -1 )) < load_sz); +relocate_kernel (load_pa); +#endif +/* * Enforce boot alignment requirements of RV32 and * RV64 by only allowing PMD or PGD mappings. @@ -387,7 +453,7 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)BUG_ON (map_size ==PAGE_SIZE );/* Sanity check alignment and size */ -BUG_ON ((PAGE_OFFSET %PGDIR_SIZE ) !=0 ); +BUILD_BUG_ON ((_AC (CONFIG_PAGE_OFFSET ,UL ) %PGDIR_SIZE ) !=0 );BUG_ON ((load_pa % map_size) !=0 );BUG_ON (load_sz >MAX_EARLY_MAPPING_SIZE );@@ -400,13 +466,13 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)create_pmd_mapping (fixmap_pmd,FIXADDR_START , (uintptr_t )fixmap_pte,PMD_SIZE ,PAGE_TABLE );/* Setup trampoline PGD and PMD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, (uintptr_t )trampoline_pmd,PGDIR_SIZE ,PAGE_TABLE ); -create_pmd_mapping (trampoline_pmd,PAGE_OFFSET , +create_pmd_mapping (trampoline_pmd, kernel_virt_addr, load_pa,PMD_SIZE ,PAGE_KERNEL_EXEC );#else /* Setup trampoline PGD */ -create_pgd_mapping (trampoline_pg_dir,PAGE_OFFSET , +create_pgd_mapping (trampoline_pg_dir, kernel_virt_addr, load_pa,PGDIR_SIZE ,PAGE_KERNEL_EXEC );#endif @@ -415,10 +481,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)* us to reach paging_init(). We map all memory banks later * in setup_vm_final() below. */ - end_va =PAGE_OFFSET + load_sz; -for (va =PAGE_OFFSET ; va < end_va; va += map_size) + end_va = kernel_virt_addr + load_sz; +for (va = kernel_virt_addr; va < end_va; va += map_size)create_pgd_mapping (early_pg_dir, va, - load_pa + (va -PAGE_OFFSET ), + load_pa + (va - kernel_virt_addr), map_size,PAGE_KERNEL_EXEC );Quoted 17 lines
Quoted 11 lines
arch/riscv/mm/init.c static void __init setup_vm_final(void) · line 528
Quoted 11 lines
arch/riscv/mm/init.c +3
-3/* Create fixed mapping for early FDT parsing */ @@ -457,9 +523,9 @@ static void __init setup_vm_final(void)break ;if (memblock_is_nomap (reg))continue ; -if (start <=__pa (PAGE_OFFSET ) && -__pa (PAGE_OFFSET ) < end) - start =__pa (PAGE_OFFSET ); +if (start <=__pa_symbol (kernel_virt_addr) && +__pa (kernel_virt_addr) < end) + start =__pa_symbol (kernel_virt_addr);Here is the same, please use
PAGE_OFFSETinstead ofkernel_virt_addr.Quoted 3 lines
Quoted 3 lines
arch/riscv/mm/init.c +0
-0map_size =best_map_size (start, end - start);for (pa = start; pa < end; pa += map_size) {Thanks,
Alex