Skip to contentSkip to conversation

x86/head/64: Build the head code as PIE

RFC PATCH In review Hou Wenlong 4 participants 15 messages Jul 12, 2023 – Nov 10, 2023

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover x86/head/64: Build the head code as PIE

During the early boot stage, the head code runs at a low identity address, which means that all absolute references would be incorrect. However, when accessing globals, the compiler does not have to generate PC-relative references. To work around this problem, every global variable access must be adjusted using fixup_pointer() in arch/x86/kernel/head64.c.
However, some global variable accesses in the current code do not use fixup_pointer(), and they may work correctly because the compiler can generate the right PC-relative references. But the behavior differs between GCC and CLANG, which has caused problems before. For example, commit c1887159eb48 ("x86/boot/64: Add missing fixup_pointer() for next_early_pgt access") stated that CLANG would generate absolute references for 'next_early_pgt' without fixup_pointer(), which leads to booting failure. Moreover, the rule is not always clear. For instance, 'pgdir_shift' is a non-static global variable similar to 'next_early_pgt', but the compiler can generate the right PC-relative reference, so fixup_pointer() is not applied to pgdir_shift when using the PGDIR_SHIFT macro.

In addition, the code in arch/x86/mm/mem_encrypt_identity.c also runs at identity address, but it uses inline assembly to use RIP-relative reference for some globals instead of fixup_pointer(). However, not all global references are changed into inline assembly.

To avoid such cases and also prepare for building the kernel as PIE, the head code could be built as PIE to force the generation of PC-relative references. This can eliminate the need for fixup_pointer() and inline assembly. However, there are still a few functions that are called by the head code but are not in head64.c and mem_encrypt_identity.c, such as snp_init() and early_snp_set_memory_shared(). Moving them into a separate compile unit and building them as PIE is a little complicated, so for now, they will remain unchanged.

Note: The change in mem_encrypt_identity.c has not been tested since I don't have the necessary environment available.

Hou Wenlong (7):
x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata x86/head/64: Add missing __head annotation to startup_64_load_idt() x86/head/64: Move all head code from head64.c into another file x86/boot/compressed: Adapt sed command if head code is built as PIE x86/head/64: Build the head code as PIE
x86/sme: Mark code as __head in mem_encrypt_identity.c x86/sme: Build the code in mem_encrypt_identity.c as PIE

 arch/x86/boot/compressed/Makefile  |   2 +-
 arch/x86/include/asm/desc.h        |  12 ++
 arch/x86/include/asm/init.h        |   2 +
 arch/x86/include/asm/mem_encrypt.h |   8 +-
 arch/x86/include/asm/setup.h       |   2 +-
 arch/x86/kernel/Makefile           |  16 +-
 arch/x86/kernel/head64.c           | 307 +----------------------------
 arch/x86/kernel/head64_identity.c  | 282 ++++++++++++++++++++++++++
 arch/x86/kernel/head_64.S          |   2 -
 arch/x86/mm/Makefile               |   3 +
 arch/x86/mm/mem_encrypt_identity.c |  58 ++----
 11 files changed, 342 insertions(+), 352 deletions(-)
 create mode 100644 arch/x86/kernel/head64_identity.c

base-commit: 1a2945f27157825a561be7840023e3664111ab2f --
2.31.1

#1 lore raw

RFC PATCH 1/7 x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

As startup_gdt and startup_gdt_descr are only used in booting, make them as __initdata to allow them to be freed after boot.

---
 arch/x86/kernel/head64.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/head64.c +2 -2
index 49f7629b17f7..dd357807ffb5 100644
@@ -69,7 +69,7 @@ EXPORT_SYMBOL(vmemmap_base);
 /*
  * GDT used on the boot CPU before switching to virtual addresses.
  */
-static struct desc_struct startup_gdt[GDT_ENTRIES] = {
+static struct desc_struct startup_gdt[GDT_ENTRIES] __initdata = {
 	[GDT_ENTRY_KERNEL32_CS]         = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
 	[GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
 	[GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),
@@ -79,7 +79,7 @@ static struct desc_struct startup_gdt[GDT_ENTRIES] = {
  * Address needs to be set at runtime because it references the startup_gdt
  * while the kernel still uses a direct mapping.
  */
-static struct desc_ptr startup_gdt_descr = {
+static struct desc_ptr startup_gdt_descr __initdata = {
 	.size = sizeof(startup_gdt),
 	.address = 0,
 };
Signature · 1 line
2.31.1
#2 lore raw

Re: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

Quoted 22 lines · Hou Wenlong

As startup_gdt and startup_gdt_descr are only used in booting, make them as __initdata to allow them to be freed after boot.

---
 arch/x86/kernel/head64.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/head64.c +1 -1
index 49f7629b17f7..dd357807ffb5 100644
@@ -69,7 +69,7 @@ EXPORT_SYMBOL(vmemmap_base);
 /*
  * GDT used on the boot CPU before switching to virtual addresses.
  */
-static struct desc_struct startup_gdt[GDT_ENTRIES] = {
+static struct desc_struct startup_gdt[GDT_ENTRIES] __initdata = {
 	[GDT_ENTRY_KERNEL32_CS]         = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
 	[GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
 	[GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),

arch/x86/kernel/head64.c static struct desc_struct startup_gdt[GDT_ENTRIES] = { · line 84

Quoted 7 lines
arch/x86/kernel/head64.c +1 -1
@@ -79,7 +79,7 @@ static struct desc_struct startup_gdt[GDT_ENTRIES] = {
  * Address needs to be set at runtime because it references the startup_gdt
  * while the kernel still uses a direct mapping.
  */
-static struct desc_ptr startup_gdt_descr = {
+static struct desc_ptr startup_gdt_descr __initdata = {
 	.size = sizeof(startup_gdt),
 	.address = 0,

Yeah, so I think this and patch #1 are independently useful of the PIE feature, and I merged them into tip:x86/boot for a v6.7 merge.

Mind re-sending any other patches for x86 that can be merged? For example patch #6 looks good too, but was mixed up a bit with a previous PIE-enablement patch. Moving the __head definition to a header should also probably done as a separate patch, followed by the widening of its use.

Thanks,

Ingo

#3 lore raw

Re: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

Quoted 36 lines · Ingo Molnar
Quoted 22 lines · Hou Wenlong

As startup_gdt and startup_gdt_descr are only used in booting, make them as __initdata to allow them to be freed after boot.

---
 arch/x86/kernel/head64.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/kernel/head64.c +1 -1
index 49f7629b17f7..dd357807ffb5 100644
@@ -69,7 +69,7 @@ EXPORT_SYMBOL(vmemmap_base);
 /*
  * GDT used on the boot CPU before switching to virtual addresses.
  */
-static struct desc_struct startup_gdt[GDT_ENTRIES] = {
+static struct desc_struct startup_gdt[GDT_ENTRIES] __initdata = {
 	[GDT_ENTRY_KERNEL32_CS]         = GDT_ENTRY_INIT(0xc09b, 0, 0xfffff),
 	[GDT_ENTRY_KERNEL_CS]           = GDT_ENTRY_INIT(0xa09b, 0, 0xfffff),
 	[GDT_ENTRY_KERNEL_DS]           = GDT_ENTRY_INIT(0xc093, 0, 0xfffff),

arch/x86/kernel/head64.c static struct desc_struct startup_gdt[GDT_ENTRIES] = { · line 84

Quoted 7 lines
arch/x86/kernel/head64.c +1 -1
@@ -79,7 +79,7 @@ static struct desc_struct startup_gdt[GDT_ENTRIES] = {
  * Address needs to be set at runtime because it references the startup_gdt
  * while the kernel still uses a direct mapping.
  */
-static struct desc_ptr startup_gdt_descr = {
+static struct desc_ptr startup_gdt_descr __initdata = {
 	.size = sizeof(startup_gdt),
 	.address = 0,

Yeah, so I think this and patch #1 are independently useful of the PIE feature, and I merged them into tip:x86/boot for a v6.7 merge.

Quoted 5 lines

Mind re-sending any other patches for x86 that can be merged? For example patch #6 looks good too, but was mixed up a bit with a previous PIE-enablement patch. Moving the __head definition to a header should also probably done as a separate patch, followed by the widening of its use.

Hi Ingo,

I have sent patch #6 separately for x86. Do you have any ideas about building the head code as PIE? Should I resend the patchset for the PIE feature?

Thanks!

Quoted 3 lines

Thanks,

Ingo

#4 lore raw

Re: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

Quoted 5 lines · Hou Wenlong

Hi Ingo,

I have sent patch #6 separately for x86. Do you have any ideas about building the head code as PIE? Should I resend the patchset for the PIE feature?

So I had a brief look, and despite reading 0/43 it was unclear to me what the precise advantages of building as PIE are.

Ie. could you please outline:

- *Exactly* how much PIE based KASLR randomization would gain us in terms of randomization granularity and effective number of randomization bits, compared to the current status quo?

- How is code generation changed at the instruction level - how does kernel size change and what are the micro-advantages/disadvantages?

- Are there any other advantages/motivation than improving KASLR?

Ie. before asking us to apply ~50 patches and add a whole new build mode and the maintainance overhead to support it into infinity and beyond, could you please offer a better list of pros and cons?

Thanks,

Ingo

#5 lore raw

Re: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

Quoted 23 lines · Ingo Molnar
Quoted 5 lines · Hou Wenlong

Hi Ingo,

I have sent patch #6 separately for x86. Do you have any ideas about building the head code as PIE? Should I resend the patchset for the PIE feature?

So I had a brief look, and despite reading 0/43 it was unclear to me what the precise advantages of building as PIE are.

Ie. could you please outline:

- *Exactly* how much PIE based KASLR randomization would gain us in terms of randomization granularity and effective number of randomization bits, compared to the current status quo?

- How is code generation changed at the instruction level - how does kernel size change and what are the micro-advantages/disadvantages?

- Are there any other advantages/motivation than improving KASLR?

Quoted 7 lines

Ie. before asking us to apply ~50 patches and add a whole new build mode and the maintainance overhead to support it into infinity and beyond, could you please offer a better list of pros and cons?

Thanks,

Ingo

If the goal is better KASLR, then what we really should spend time on was Kristen Accardi's fgKASLR patches, which not only exponentially(!) increases the randomization entrophy but also *actually* avoids the "one leak and it's over" problem.

However, she gave up on it because she got no interest, despite working code.

#6 lore raw

Re: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

Quoted 4 lines · H. Peter Anvin

If the goal is better KASLR, then what we really should spend time on was Kristen Accardi's fgKASLR patches, which not only exponentially(!) increases the randomization entrophy but also *actually* avoids the "one leak and it's over" problem.

Agreed. Going by this version of function-granularity KASLR from 3 years ago:

https://lwn.net/Articles/824307/
https://lwn.net/ml/linux-kernel/[email protected]/

The fgKASLR feature looks entirely viable to me. Back then I presumed it would get iterated beyond v3, and then it fell off my radar. :-/

If Kristen or someone else would like to dust this off & submit a fresh version it would be much appreciated!

Thanks,

Ingo

#7 lore raw

Re: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

Quoted 11 lines · Ingo Molnar
Quoted 4 lines · H. Peter Anvin

If the goal is better KASLR, then what we really should spend time on was Kristen Accardi's fgKASLR patches, which not only exponentially(!) increases the randomization entrophy but also *actually* avoids the "one leak and it's over" problem.

Agreed. Going by this version of function-granularity KASLR from 3 years ago:

Quoted 8 lines

https://lwn.net/Articles/824307/
https://lwn.net/ml/linux-kernel/[email protected]/

The fgKASLR feature looks entirely viable to me. Back then I presumed it would get iterated beyond v3, and then it fell off my radar. :-/

If Kristen or someone else would like to dust this off & submit a fresh version it would be much appreciated!

That actually got up to v10:

https://lkml.kernel.org/lkml/[email protected]

Anyway, I'm also very interested in this. If nobody else is working on it then I could give it a try.

BTW I've wondered if translation-unit granularity would be more preferable than function granularity due to improved i-cache locality and (possibly) easier livepatch compatibility.

Signature · 1 line
Josh
#8 lore raw

Re: [PATCH RFC 1/7] x86/head/64: Mark startup_gdt and startup_gdt_descr as __initdata

In reply to Ingo Molnar
Quoted 18 lines · Ingo Molnar
Quoted 5 lines · Hou Wenlong

Hi Ingo,

I have sent patch #6 separately for x86. Do you have any ideas about building the head code as PIE? Should I resend the patchset for the PIE feature?

So I had a brief look, and despite reading 0/43 it was unclear to me what the precise advantages of building as PIE are.

Ie. could you please outline:

- *Exactly* how much PIE based KASLR randomization would gain us in terms of randomization granularity and effective number of randomization bits, compared to the current status quo?

Quoted 9 lines

- How is code generation changed at the instruction level - how does kernel size change and what are the micro-advantages/disadvantages?

- Are there any other advantages/motivation than improving KASLR?

Ie. before asking us to apply ~50 patches and add a whole new build mode and the maintainance overhead to support it into infinity and beyond, could you please offer a better list of pros and cons?

Hi Ingo,

Thanks for your reply. I apologize for the confusion. Waht I meant to say is that I would like to resend the remaining part of this patchset that building the head code as PIE. As mentioned in the cover letter, building the head code as PIE can eliminate certain workarounds such as the "fixup_pointer" in head64.c and the inline assembly in mem_encrypt_identity.c. This is considered a cleanup. However, it is still necessary to use inline assembly to obtain the absolute symbol value during the pagetable building process.

Regarding the entire PIE patchset, I agree that it is complex and there are no obvious use cases apart from improving KASLR. As mentioned earlier, the primary motivation is to increase the flexibility of the kernel image address rather than prioritizing security, enabling the kernel image to be placed at any virtual address. The use cases in our internal environment are specific and not widespread, so we do not feel an urgent need to push it forward at the moment.

Thanks!

Quoted 3 lines

Thanks,

Ingo

#9 lore raw

RFC PATCH 2/7 x86/head/64: Add missing __head annotation to startup_64_load_idt()

In reply to Hou Wenlong

This function is currently only used in the head code and is only called from startup_64_setup_env(). Although it would be inlined by the compiler, it would be better to mark it as __head too.

---
 arch/x86/kernel/head64.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/kernel/head64.c +1 -1
index dd357807ffb5..5066189b00ac 100644
@@ -588,7 +588,7 @@ static void set_bringup_idt_handler(gate_desc *idt, int n, void *handler)
 }
 
 /* This runs while still in the direct mapping */
-static void startup_64_load_idt(unsigned long physbase)
+static void __head startup_64_load_idt(unsigned long physbase)
 {
 	struct desc_ptr *desc = fixup_pointer(&bringup_idt_descr, physbase);
 	gate_desc *idt = fixup_pointer(bringup_idt_table, physbase);
Signature · 1 line
2.31.1
#10 lore raw

RFC PATCH 3/7 x86/head/64: Move all head code from head64.c into another file

In reply to Hou Wenlong

This is preparation for building the head code as PIE and getting rid of all fixup_pointer() calls.

---
 arch/x86/include/asm/desc.h       |  12 ++
 arch/x86/kernel/Makefile          |   5 +-
 arch/x86/kernel/head64.c          | 307 +----------------------------
 arch/x86/kernel/head64_identity.c | 315 ++++++++++++++++++++++++++++++
 4 files changed, 334 insertions(+), 305 deletions(-)
 create mode 100644 arch/x86/kernel/head64_identity.c
arch/x86/include/asm/desc.h +12 -0
index ab97b22ac04a..b568331b0caa 100644
@@ -431,6 +431,18 @@ static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d)
 #endif
 }
 
+static inline void set_bringup_idt_handler(gate_desc *idt, int n, void *handler)
+{
+#ifdef CONFIG_AMD_MEM_ENCRYPT
+	struct idt_data data;
+	gate_desc desc;
+
+	init_idt_data(&data, n, handler);
+	idt_init_desc(&desc, &data);
+	native_write_idt_entry(idt, n, &desc);
+#endif
+}
+
 extern unsigned long system_vectors[];
 
 extern void load_current_idt(void);
arch/x86/kernel/Makefile +4 -1
index 4070a01c11b7..2fd9a4fe27b1 100644
@@ -16,11 +16,13 @@ CFLAGS_REMOVE_kvmclock.o = -pg
 CFLAGS_REMOVE_ftrace.o = -pg
 CFLAGS_REMOVE_early_printk.o = -pg
 CFLAGS_REMOVE_head64.o = -pg
+CFLAGS_REMOVE_head64_identity.o = -pg
 CFLAGS_REMOVE_sev.o = -pg
 CFLAGS_REMOVE_rethook.o = -pg
 endif
 
 KASAN_SANITIZE_head$(BITS).o				:= n
+KASAN_SANITIZE_head64_identity.o			:= n
 KASAN_SANITIZE_dumpstack.o				:= n
 KASAN_SANITIZE_dumpstack_$(BITS).o			:= n
 KASAN_SANITIZE_stacktrace.o				:= n
@@ -31,6 +33,7 @@ KASAN_SANITIZE_sev.o					:= n
 # by several compilation units. To be safe, disable all instrumentation.
 KCSAN_SANITIZE := n
 KMSAN_SANITIZE_head$(BITS).o				:= n
+KMSAN_SANITIZE_head64_identity.o			:= n
 KMSAN_SANITIZE_nmi.o					:= n
 
 # If instrumentation of this dir is enabled, boot hangs during first second.
@@ -153,5 +156,5 @@ ifeq ($(CONFIG_X86_64),y)
 	obj-$(CONFIG_GART_IOMMU)	+= amd_gart_64.o aperture_64.o
 
 	obj-$(CONFIG_MMCONF_FAM10H)	+= mmconf-fam10h_64.o
-	obj-y				+= vsmp_64.o
+	obj-y				+= vsmp_64.o head64_identity.o
 endif
arch/x86/kernel/head64.c +3 -304

333 lines not on this page · raw message

arch/x86/kernel/head64_identity.c new file +315 -0

315 lines not on this page · raw message

Signature · 1 line
2.31.1
#11 lore raw

RFC PATCH 4/7 x86/boot/compressed: Adapt sed command if head code is built as PIE

In reply to Hou Wenlong

When x86/head64_identity.c is built as PIE, all symbols would be set as hidden to omit GOT references. According to the generic ABI, a hidden symbol contained in a relocatable object must be either removed or converted to STB_LOCAL binding by the link-editor when the relocatable object is included in an executable file or shared object. Both gold and ld.lld change the binding of a STV_HIDDEND symbol to STB_LOCAL. However, for GNU ld, it will keep the global hidden. The sed command to generate voffset.h only captures global symbols, so an empty voffset.h would be generated with ld.lld. Therefore, local symbols should also be captured in the sed command.

---
 arch/x86/boot/compressed/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/x86/boot/compressed/Makefile +1 -1
index 40d2ff503079..ca57ad0d2d22 100644
@@ -79,7 +79,7 @@ LDFLAGS_vmlinux += -T
 hostprogs	:= mkpiggy
 HOST_EXTRACFLAGS += -I$(srctree)/tools/include
 
-sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p'
+sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVWabcdgrstvw] \(_text\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p'
 
 quiet_cmd_voffset = VOFFSET $@
       cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@
Signature · 1 line
2.31.1
#12 lore raw

RFC PATCH 5/7 x86/head/64: Build the head code as PIE

In reply to Hou Wenlong

During the early boot stage, the head code runs in a low identity address, so all absolute references would be incorrect. However, the compiler doesn't have to generate PC-relative references when accessing globals. To work around this problem, every global variable access must be adjusted using fixup_pointer(). Nevertheless, the compiler could generate absolute references for some global variable accesses, and the behavior differs between GCC and CLANG. For example, GCC generates PC-relative references for 'next_early_pgt', while CLANG generates absolute references. Moreover, the rule is not always clear, e.g., 'pgdir_shift' is a non-static global variable similar to 'next_early_pgt', but the compiler could generate the right PC-relative reference, so fixup_pointer() is not applied when using the PGDIR_SHIFT macro.

To avoid such cases, build the head code as PIE to force the generation of PC-relative references and eliminate the need for fixup_pointer(). However, it may be necessary to obtain an absolute virtual address for some symbols in the head code. In this case, use 'movabsq'. Additionally, the 'mcmodel=kernel' option is not compatible with the 'fPIE' option and needs to be removed. This will result in using the '%fs' register for stack canary access if the stack protector is enabled. The 'mstack-protector-guard-reg' compiler option could be used to fix this, but it is only used in 32-bit now and 64-bit is still using the fixed location version. Since the head code is in the early booting stage, it is safe to disable the stack protector for the head code.

---
 arch/x86/include/asm/setup.h      |   2 +-
 arch/x86/kernel/Makefile          |  11 +++
 arch/x86/kernel/head64_identity.c | 112 +++++++++++-------------------
 arch/x86/kernel/head_64.S         |   2 -
 4 files changed, 52 insertions(+), 75 deletions(-)
arch/x86/include/asm/setup.h +1 -1
index f3495623ac99..b893b0cdddac 100644
@@ -50,7 +50,7 @@ extern unsigned long saved_video_mode;
 extern void reserve_standard_io_resources(void);
 extern void i386_reserve_resources(void);
 extern unsigned long __startup_64(unsigned long physaddr, struct boot_params *bp);
-extern void startup_64_setup_env(unsigned long physbase);
+extern void startup_64_setup_env(void);
 extern void early_setup_idt(void);
 extern void __init do_early_exception(struct pt_regs *regs, int trapnr);
 
arch/x86/kernel/Makefile +11 -0
index 2fd9a4fe27b1..6564113f5298 100644
@@ -44,6 +44,17 @@ KCOV_INSTRUMENT		:= n
 
 CFLAGS_irq.o := -I $(srctree)/$(src)/../include/asm/trace
 
+# The 'mcmodel=kernel' option is not compatible with the 'fPIE' option and
+# needs to be removed. This will result in using the '%fs' register for stack
+# canary access if the stack protector is enabled. The
+# 'mstack-protector-guard-reg' compiler option could be used to fix this, but
+# it is only used in 32-bit now and 64-bit is still using the fixed location
+# version. Since the head code is in the early booting stage, it is safe to
+# disable the stack protector for the head code.
+CFLAGS_REMOVE_head64_identity.o += -mcmodel=kernel
+CFLAGS_head64_identity.o += -fPIE -include $(srctree)/include/linux/hidden.h
+CFLAGS_head64_identity.o += -fno-stack-protector
+
 obj-y			+= head_$(BITS).o
 obj-y			+= head$(BITS).o
 obj-y			+= ebda.o
arch/x86/kernel/head64_identity.c +40 -72

230 lines not on this page · raw message

arch/x86/kernel/head_64.S +0 -2
index c5b9289837dc..5b46da66d6c8 100644
@@ -66,8 +66,6 @@ SYM_CODE_START_NOALIGN(startup_64)
 	/* Set up the stack for verify_cpu() */
 	leaq	(__end_init_task - PTREGS_SIZE)(%rip), %rsp
 
-	leaq	_text(%rip), %rdi
-
 	/* Setup GSBASE to allow stack canary access for C code */
 	movl	$MSR_GS_BASE, %ecx
 	leaq	INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
Signature · 1 line
2.31.1
#13 lore raw

RFC PATCH 6/7 x86/sme: Mark code as __head in mem_encrypt_identity.c

In reply to Hou Wenlong

The functions sme_enable() and sme_encrypt_kernel() are only called by the head code which runs in identity virtual address. Therefore, it's better to mark them as __head as well.

---
 arch/x86/include/asm/init.h        |  2 ++
 arch/x86/include/asm/mem_encrypt.h |  8 ++++----
 arch/x86/kernel/head64_identity.c  |  3 +--
 arch/x86/mm/mem_encrypt_identity.c | 27 ++++++++++++++-------------
 4 files changed, 21 insertions(+), 19 deletions(-)
arch/x86/include/asm/init.h +2 -0
index 5f1d3c421f68..cc9ccf61b6bd 100644
@@ -2,6 +2,8 @@
 #ifndef _ASM_X86_INIT_H
 #define _ASM_X86_INIT_H
 
+#define __head	__section(".head.text")
+
 struct x86_mapping_info {
 	void *(*alloc_pgt_page)(void *); /* allocate buf for page table */
 	void *context;			 /* context for alloc_pgt_page */
arch/x86/include/asm/mem_encrypt.h +4 -4
index 7f97a8a97e24..7b6616f67398 100644
@@ -45,8 +45,8 @@ void __init sme_unmap_bootdata(char *real_mode_data);
 void __init sme_early_init(void);
 void __init sev_setup_arch(void);
 
-void __init sme_encrypt_kernel(struct boot_params *bp);
-void __init sme_enable(struct boot_params *bp);
+void sme_encrypt_kernel(struct boot_params *bp);
+void sme_enable(struct boot_params *bp);
 
 int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size);
 int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
@@ -75,8 +75,8 @@ static inline void __init sme_unmap_bootdata(char *real_mode_data) { }
 static inline void __init sme_early_init(void) { }
 static inline void __init sev_setup_arch(void) { }
 
-static inline void __init sme_encrypt_kernel(struct boot_params *bp) { }
-static inline void __init sme_enable(struct boot_params *bp) { }
+static inline void sme_encrypt_kernel(struct boot_params *bp) { }
+static inline void sme_enable(struct boot_params *bp) { }
 
 static inline void sev_es_init_vc_handling(void) { }
 
arch/x86/kernel/head64_identity.c +1 -2
index 93f5831917bc..521fef98cccd 100644
@@ -19,6 +19,7 @@
 #include <asm/sections.h>
 #include <asm/trapnr.h>
 #include <asm/sev.h>
+#include <asm/init.h>
 
 extern pmd_t early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
 extern unsigned int next_early_pgt;
@@ -43,8 +44,6 @@ static struct desc_ptr startup_gdt_descr __initdata = {
 	.address = 0,
 };
 
-#define __head __section(".head.text")
-
 #ifdef CONFIG_X86_5LEVEL
 static bool __head check_la57_support(void)
 {
arch/x86/mm/mem_encrypt_identity.c +14 -13
index d73aeb16417f..72aeb0f3dec6 100644
@@ -46,6 +46,7 @@
 #include <asm/cmdline.h>
 #include <asm/coco.h>
 #include <asm/sev.h>
+#include <asm/init.h>
 
 #include "mm_internal.h"
 
@@ -99,7 +100,7 @@ static char sme_cmdline_arg[] __initdata = "mem_encrypt";
 static char sme_cmdline_on[]  __initdata = "on";
 static char sme_cmdline_off[] __initdata = "off";
 
-static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
+static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
 {
 	unsigned long pgd_start, pgd_end, pgd_size;
 	pgd_t *pgd_p;
@@ -114,7 +115,7 @@ static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
 	memset(pgd_p, 0, pgd_size);
 }
 
-static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
+static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
 {
 	pgd_t *pgd;
 	p4d_t *p4d;
@@ -151,7 +152,7 @@ static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
 	return pud;
 }
 
-static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
+static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
 {
 	pud_t *pud;
 	pmd_t *pmd;
@@ -167,7 +168,7 @@ static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
 	set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags));
 }
 
-static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
+static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
 {
 	pud_t *pud;
 	pmd_t *pmd;
@@ -193,7 +194,7 @@ static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
 		set_pte(pte, __pte(ppd->paddr | ppd->pte_flags));
 }
 
-static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
+static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
 {
 	while (ppd->vaddr < ppd->vaddr_end) {
 		sme_populate_pgd_large(ppd);
@@ -203,7 +204,7 @@ static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
 	}
 }
 
-static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
+static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
 {
 	while (ppd->vaddr < ppd->vaddr_end) {
 		sme_populate_pgd(ppd);
@@ -213,7 +214,7 @@ static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
 	}
 }
 
-static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
+static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
 				   pmdval_t pmd_flags, pteval_t pte_flags)
 {
 	unsigned long vaddr_end;
@@ -237,22 +238,22 @@ static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
 	__sme_map_range_pte(ppd);
 }
 
-static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
+static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
 {
 	__sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
 }
 
-static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
+static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
 {
 	__sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
 }
 
-static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
+static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
 {
 	__sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
 }
 
-static unsigned long __init sme_pgtable_calc(unsigned long len)
+static unsigned long __head sme_pgtable_calc(unsigned long len)
 {
 	unsigned long entries = 0, tables = 0;
 
@@ -289,7 +290,7 @@ static unsigned long __init sme_pgtable_calc(unsigned long len)
 	return entries + tables;
 }
 
-void __init sme_encrypt_kernel(struct boot_params *bp)
+void __head sme_encrypt_kernel(struct boot_params *bp)
 {
 	unsigned long workarea_start, workarea_end, workarea_len;
 	unsigned long execute_start, execute_end, execute_len;
@@ -502,7 +503,7 @@ void __init sme_encrypt_kernel(struct boot_params *bp)
 	native_write_cr3(__native_read_cr3());
 }
 
-void __init sme_enable(struct boot_params *bp)
+void __head sme_enable(struct boot_params *bp)
 {
 	const char *cmdline_ptr, *cmdline_arg, *cmdline_on, *cmdline_off;
 	unsigned int eax, ebx, ecx, edx;
Signature · 1 line
2.31.1
#14 lore raw

RFC PATCH 7/7 x86/sme: Build the code in mem_encrypt_identity.c as PIE

In reply to Hou Wenlong

Similar to head64.c, all the code in mem_encrypt_identity.c runs in identity address. However, it uses inline assembly to use RIP-relative reference for some globals. Therefore, build the code as PIE to force the compiler to generate RIP-relative reference, allowing for all inline assembly to be removed.

---
The changes have not been tested on AMD SME.

 arch/x86/mm/Makefile               |  3 +++
 arch/x86/mm/mem_encrypt_identity.c | 31 +++++-------------------------
 2 files changed, 8 insertions(+), 26 deletions(-)
arch/x86/mm/Makefile +3 -0
index c80febc44cd2..f5d7b22c5f1b 100644
@@ -32,6 +32,9 @@ obj-y				+= pat/
 # Make sure __phys_addr has no stackprotector
 CFLAGS_physaddr.o		:= -fno-stack-protector
 CFLAGS_mem_encrypt_identity.o	:= -fno-stack-protector
+CFLAGS_mem_encrypt_identity.o 	+= -fPIE -include $(srctree)/include/linux/hidden.h
+
+CFLAGS_REMOVE_mem_encrypt_identity.o += -mcmodel=kernel
 
 CFLAGS_fault.o := -I $(srctree)/$(src)/../include/asm/trace
 
arch/x86/mm/mem_encrypt_identity.c +5 -26
index 72aeb0f3dec6..2f292ab4e6a9 100644
@@ -343,13 +343,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 	}
 #endif
 
-	/*
-	 * We're running identity mapped, so we must obtain the address to the
-	 * SME encryption workarea using rip-relative addressing.
-	 */
-	asm ("lea sme_workarea(%%rip), %0"
-	     : "=r" (workarea_start)
-	     : "p" (sme_workarea));
+	workarea_start = (unsigned long)(void *)sme_workarea;
 
 	/*
 	 * Calculate required number of workarea bytes needed:
@@ -505,7 +499,7 @@ void __head sme_encrypt_kernel(struct boot_params *bp)
 
 void __head sme_enable(struct boot_params *bp)
 {
-	const char *cmdline_ptr, *cmdline_arg, *cmdline_on, *cmdline_off;
+	const char *cmdline_ptr;
 	unsigned int eax, ebx, ecx, edx;
 	unsigned long feature_mask;
 	bool active_by_default;
@@ -578,21 +572,6 @@ void __head sme_enable(struct boot_params *bp)
 		goto out;
 	}
 
-	/*
-	 * Fixups have not been applied to phys_base yet and we're running
-	 * identity mapped, so we must obtain the address to the SME command
-	 * line argument data using rip-relative addressing.
-	 */
-	asm ("lea sme_cmdline_arg(%%rip), %0"
-	     : "=r" (cmdline_arg)
-	     : "p" (sme_cmdline_arg));
-	asm ("lea sme_cmdline_on(%%rip), %0"
-	     : "=r" (cmdline_on)
-	     : "p" (sme_cmdline_on));
-	asm ("lea sme_cmdline_off(%%rip), %0"
-	     : "=r" (cmdline_off)
-	     : "p" (sme_cmdline_off));
-
 	if (IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT))
 		active_by_default = true;
 	else
@@ -601,12 +580,12 @@ void __head sme_enable(struct boot_params *bp)
 	cmdline_ptr = (const char *)((u64)bp->hdr.cmd_line_ptr |
 				     ((u64)bp->ext_cmd_line_ptr << 32));
 
-	if (cmdline_find_option(cmdline_ptr, cmdline_arg, buffer, sizeof(buffer)) < 0)
+	if (cmdline_find_option(cmdline_ptr, sme_cmdline_arg, buffer, sizeof(buffer)) < 0)
 		return;
 
-	if (!strncmp(buffer, cmdline_on, sizeof(buffer)))
+	if (!strncmp(buffer, sme_cmdline_on, sizeof(buffer)))
 		sme_me_mask = me_mask;
-	else if (!strncmp(buffer, cmdline_off, sizeof(buffer)))
+	else if (!strncmp(buffer, sme_cmdline_off, sizeof(buffer)))
 		sme_me_mask = 0;
 	else
 		sme_me_mask = active_by_default ? me_mask : 0;

--
2.31.1