x86/entry: Convert error_entry() to C code
Subsystems (2)
cover x86/entry: Convert error_entry() to C code
From: Lai Jiangshan <[email protected]>
Add some C equivalent functions of the ASM macros and implement the whole error_entry() as C code.
The patches are picked from the huge patchset
https://lore.kernel.org/lkml/[email protected]/
which converts a large chunk of ASM code to C code.
The C version generally has better readability and easier to be updated/improved.
This smaller patchset converts error_entry() only.
The equivalent ASM macros are not removed because they are still used by the IST exceptions.
Changed from the old huge patchset:
squash some patches
Lai Jiangshan (7):
x86/entry: Introduce __entry_text for entry code written in C x86/entry: Move PTI_USER_* to arch/x86/include/asm/processor-flags.h
x86: Mark __native_read_cr3() & native_write_cr3() as __always_inline x86/entry: Add arch/x86/entry/entry64.c for C entry code x86/entry: Add the C verion of SWITCH_TO_KERNEL_CR3 as
switch_to_kernel_cr3()
x86/traps: Add fence_swapgs_{user,kernel}_entry() and
user_entry_swapgs_and_fence()
x86/entry: Implement the whole error_entry() as C code
arch/x86/entry/Makefile | 3 +- arch/x86/entry/calling.h | 10 -- arch/x86/entry/entry64.c | 137 +++++++++++++++++++++++++ arch/x86/entry/entry_64.S | 85 +-------------- arch/x86/include/asm/idtentry.h | 3 + arch/x86/include/asm/pgtable.h | 23 +---- arch/x86/include/asm/processor-flags.h | 15 +++ arch/x86/include/asm/proto.h | 1 + arch/x86/include/asm/special_insns.h | 4 +- arch/x86/include/asm/traps.h | 1 + arch/x86/kernel/traps.c | 2 - include/linux/compiler_types.h | 8 +- 12 files changed, 172 insertions(+), 120 deletions(-) create mode 100644 arch/x86/entry/entry64.c
Signature · 1 line
2.19.1.6.gb485710b
PATCH 1/7 x86/entry: Introduce __entry_text for entry code written in C
From: Lai Jiangshan <[email protected]>
Some entry code will be implemented in C files. Introduce __entry_text to set them in .entry.text section.
The new __entry_text disables instrumentation like noinstr, so __noinstr_section() is added for noinstr and the new __entry_text.
Note, entry code can not access to %gs before the %gs base is switched to kernel %gs base, so stack protector can not be used on the C entry code. But __entry_text doesn't disable stack protector since some compilers might not support function level granular attribute to disable stack protector. It will be disabled in C file level.
--- arch/x86/include/asm/idtentry.h | 3 +++ include/linux/compiler_types.h | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-)
arch/x86/include/asm/idtentry.h +3 -0
@@ -13,6 +13,9 @@#include <asm/irq_stack.h> +/* Entry code written in C. */ +#define __entry_text __noinstr_section(".entry.text") +/** * DECLARE_IDTENTRY - Declare functions for simple IDT entry points * No error code pushed by hardware
include/linux/compiler_types.h +5 -3
@@ -225,9 +225,11 @@ struct ftrace_likely_data {#endif /* Section for code which can't be instrumented at all */ -#define noinstr \ - noinline notrace __attribute((__section__(".noinstr.text"))) \ - __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage +#define __noinstr_section (section) \ + noinline notrace __section(section) __no_profile \ + __no_kcsan __no_sanitize_address __no_sanitize_coverage + +#define noinstr __noinstr_section(".noinstr.text")#endif /* __KERNEL__ */
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH 1/7] x86/entry: Introduce __entry_text for entry code written in C
Quoted 46 lines · Lai Jiangshan
From: Lai Jiangshan <[email protected]>
Some entry code will be implemented in C files. Introduce
__entry_textto set them in .entry.text section.The new
__entry_textdisables instrumentation like noinstr, so__noinstr_section()is added for noinstr and the new__entry_text.Note, entry code can not access to %gs before the %gs base is switched to kernel %gs base, so stack protector can not be used on the C entry code. But
__entry_textdoesn't disable stack protector since some compilers might not support function level granular attribute to disable stack protector. It will be disabled in C file level.Cc Borislav Petkov <[email protected]>Reviewed-by Miguel Ojeda <[email protected]>Reviewed-by Kees Cook <[email protected]>Suggested-by Nick Desaulniers <[email protected]>Suggested-by Peter Zijlstra <[email protected]>Signed-off-by Lai Jiangshan <[email protected]>--- arch/x86/include/asm/idtentry.h | 3 +++ include/linux/compiler_types.h | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-)arch/x86/include/asm/idtentry.h +3
-0@@ -13,6 +13,9 @@#include <asm/irq_stack.h> +/* Entry code written in C. */ +#define __entry_text __noinstr_section(".entry.text") +/** * DECLARE_IDTENTRY - Declare functions for simple IDT entry points * No error code pushed by hardware include/linux/compiler_types.h +0
-0@@ -225,9 +225,11 @@ struct ftrace_likely_data {#endif
include/linux/compiler_types.h struct ftrace_likely_data { · line 232
Quoted 10 lines
include/linux/compiler_types.h +5
-3/* Section for code which can't be instrumented at all */ -#define noinstr \ - noinline notrace __attribute((__section__(".noinstr.text"))) \ - __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage +#define __noinstr_section (section) \ + noinline notrace __section(section) __no_profile \ + __no_kcsan __no_sanitize_address __no_sanitize_coverage + +#define noinstr __noinstr_section(".noinstr.text")
I haven't looked at the rest of the series, but isn't noinstr used in a bunch of places? Wont this commit break all those uses or at least make it break bisection?
Also, my suggestion was simply not to open code
__attribute__((section(""))). Does this work?
```
diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index 7924f27f5c8b..10ec7039e17d 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -13,6 +13,9 @@
#include <asm/irq_stack.h>
+/* Entry code written in C. Override the section used by noinstr. */
+#define __entry_text noinstr __section(".entry.text")
+
/**
* DECLARE_IDTENTRY - Declare functions for simple IDT entry points
* No error code pushed by hardware
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index 1c2c33ae1b37..ce623099eb21 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -226,8 +226,8 @@ struct ftrace_likely_data {
/* Section for code which can't be instrumented at all */
#define noinstr
\
- noinline notrace __attribute((__section__(".noinstr.text"))) \
- __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage
+ noinline notrace __section(".noinstr.text") __no_kcsan \
+ __no_sanitize_address __no_profile __no_sanitize_coverage
#endif /* __KERNEL__ */
```Signature · 2 lines
Thanks, ~Nick Desaulniers
Re: [PATCH 1/7] x86/entry: Introduce __entry_text for entry code written in C
Quoted 3 lines · Nick Desaulniers
I haven't looked at the rest of the series, but isn't
noinstrused in a bunch of places? Wont this commit break all those uses or at
Except the order, it expands to the same, no? Or what is the issue?
Cheers,
Miguel
Re: [PATCH 1/7] x86/entry: Introduce __entry_text for entry code written in C
Quoted 8 lines · Miguel Ojeda
Quoted 3 lines · Nick Desaulniers
I haven't looked at the rest of the series, but isn't
noinstrused in a bunch of places? Wont this commit break all those uses or atExcept the order, it expands to the same, no? Or what is the issue?
Yeah, nvm, I missed the bottom #define for some reason.
Signature · 2 lines
Thanks, ~Nick Desaulniers
Re: [PATCH 1/7] x86/entry: Introduce __entry_text for entry code written in C
Quoted 2 lines · Nick Desaulniers
Yeah, nvm, I missed the bottom #define for some reason.
No worries, I got worried there could be a change of behavior I did not know about the order of those... (in which case we should really document it! :)
Cheers,
Miguel
Re: [PATCH 1/7] x86/entry: Introduce __entry_text for entry code written in C
In reply to Nick DesaulniersQuoted 14 lines · Nick Desaulniers
include/linux/compiler_types.h struct ftrace_likely_data { · line 232
Quoted 9 lines · Lai Jiangshan
/* Section for code which can't be instrumented at all */
include/linux/compiler_types.h +5
-3-#define noinstr \ - noinline notrace __attribute((__section__(".noinstr.text"))) \ - __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage +#define __noinstr_section (section) \ + noinline notrace __section(section) __no_profile \ + __no_kcsan __no_sanitize_address __no_sanitize_coverage + +#define noinstr __noinstr_section(".noinstr.text")I haven't looked at the rest of the series, but isn't
noinstrused in a bunch of places? Wont this commit break all those uses or at least make it break bisection?
Why so?
This is still applying all the 'no' annotations and puts the code in the very same section as before, no?
Thanks,
tglx
PATCH 2/7 x86/entry: Move PTI_USER_* to arch/x86/include/asm/processor-flags.h
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
These constants will be also used in C file.
Move them to arch/x86/include/asm/processor-flags.h which already has a kin X86_CR3_PTI_PCID_USER_BIT defined in it.
Remove PTI_PGTABLE_SWITCH_BIT and replace it with PTI_USER_PGTABLE_BIT since they are the same in meaning and value.
Remove kernel_to_user_p4dp() and user_to_kernel_p4dp() since they are using the removed PTI_PGTABLE_SWITCH_BIT and they have no caller.
--- arch/x86/entry/calling.h | 10 ---------- arch/x86/include/asm/pgtable.h | 23 +++-------------------- arch/x86/include/asm/processor-flags.h | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 30 deletions(-)
arch/x86/entry/calling.h +0 -10
@@ -142,16 +142,6 @@ For 32-bit we have the following conventions - kernel is built with#ifdef CONFIG_PAGE_TABLE_ISOLATION -/* -* PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two -* halves: -*/ -#define PTI_USER_PGTABLE_BIT PAGE_SHIFT -#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) -#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT -#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) -#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGTABLE_MASK) - .macro SET_NOFLUSH_BIT reg:req bts $X86_CR3_PCID_NOFLUSH_BIT, \reg .endm
arch/x86/include/asm/pgtable.h +3 -20
@@ -5,6 +5,7 @@#include <linux/mem_encrypt.h> #include <asm/page.h> #include <asm/pgtable_types.h> +#include <asm/processor-flags.h> /* * Macro to mark a page protection value as UC-
@@ -1191,14 +1192,6 @@ static inline bool pgdp_maps_userspace(void *__ptr)static inline int pgd_large (pgd_t pgd) {return 0 ; }#ifdef CONFIG_PAGE_TABLE_ISOLATION -/* -* All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages -* (8k-aligned and 8k in size). The kernel one is at the beginning 4k and -* the user one is in the last 4k. To switch between them, you -* just need to flip the 12th bit in their addresses. -*/ -#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT -/* * This generates better code than the inline assembly in * __set_bit().
@@ -1220,22 +1213,12 @@ static inline void *ptr_clear_bit(void *ptr, int bit)static inline pgd_t *kernel_to_user_pgdp (pgd_t *pgdp) { -return ptr_set_bit (pgdp,PTI_PGTABLE_SWITCH_BIT ); +return ptr_set_bit (pgdp,PTI_USER_PGTABLE_BIT ); }static inline pgd_t *user_to_kernel_pgdp (pgd_t *pgdp) { -return ptr_clear_bit (pgdp,PTI_PGTABLE_SWITCH_BIT ); -} - -static inline p4d_t *kernel_to_user_p4dp (p4d_t *p4dp) -{ -return ptr_set_bit (p4dp,PTI_PGTABLE_SWITCH_BIT ); -} - -static inline p4d_t *user_to_kernel_p4dp (p4d_t *p4dp) -{ -return ptr_clear_bit (p4dp,PTI_PGTABLE_SWITCH_BIT ); +return ptr_clear_bit (pgdp,PTI_USER_PGTABLE_BIT ); }#endif /* CONFIG_PAGE_TABLE_ISOLATION */
arch/x86/include/asm/processor-flags.h +15 -0
@@ -4,6 +4,7 @@#include <uapi/asm/processor-flags.h> #include <linux/mem_encrypt.h> +#include <asm/page_types.h> #ifdef CONFIG_VM86 #define X86_VM_MASK X86_EFLAGS_VM
@@ -50,7 +51,21 @@#endif #ifdef CONFIG_PAGE_TABLE_ISOLATION +# define X86_CR3_PTI_PCID_USER_BIT 11 + +#ifdef CONFIG_X86_64 +/* +* PAGE_TABLE_ISOLATION PGDs are 8k. Flip bit 12 to switch between the two +* halves: +*/ +#define PTI_USER_PGTABLE_BIT PAGE_SHIFT +#define PTI_USER_PGTABLE_MASK (1 << PTI_USER_PGTABLE_BIT) +#define PTI_USER_PCID_BIT X86_CR3_PTI_PCID_USER_BIT +#define PTI_USER_PCID_MASK (1 << PTI_USER_PCID_BIT) +#define PTI_USER_PGTABLE_AND_PCID_MASK (PTI_USER_PCID_MASK | PTI_USER_PGTABLE_MASK) +#endif +#endif #endif /* _ASM_X86_PROCESSOR_FLAGS_H */
Signature · 1 line
2.19.1.6.gb485710b
Re: [PATCH 2/7] x86/entry: Move PTI_USER_* to arch/x86/include/asm/processor-flags.h
Quoted 9 lines · Lai Jiangshan
From: Lai Jiangshan <[email protected]>
These constants will be also used in C file.
Move them to
arch/x86/include/asm/processor-flags.hwhich already has a kinX86_CR3_PTI_PCID_USER_BITdefined in it.Remove
PTI_PGTABLE_SWITCH_BITand replace it withPTI_USER_PGTABLE_BITsince they are the same in meaning and value.
Just from looking at the patch it's obvious that it breaks the 32bit CONFIG_PAGETABLE_ISOLATION=y build. Sigh.
Thanks,
tglx
Re: [PATCH 2/7] x86/entry: Move PTI_USER_* to arch/x86/include/asm/processor-flags.h
Quoted 6 lines · Thomas Gleixner
Quoted 4 lines · Lai Jiangshan
From: Lai Jiangshan <[email protected]>
These constants will be also used in C file.
Quoted 8 lines
Quoted 5 lines
Move them to
arch/x86/include/asm/processor-flags.hwhich already has a kinX86_CR3_PTI_PCID_USER_BITdefined in it.Remove
PTI_PGTABLE_SWITCH_BITand replace it withPTI_USER_PGTABLE_BITsince they are the same in meaning and value.Just from looking at the patch it's obvious that it breaks the 32bit
CONFIG_PAGETABLE_ISOLATION=ybuild. Sigh.
Ahhh.
Thank you!
I'm sorry and I will move the "#ifdef CONFIG_X86_64" down a little bit in arch/x86/include/asm/processor-flags.h.
Thanks
Lai
PATCH 3/7 x86: Mark __native_read_cr3() & native_write_cr3() as __always_inline
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Mark __native_read_cr3() & native_write_cr3() as __always_inline to ensure they are not instrumentable and in the .entry.text section if the caller is not instrumentable and in the .entry.text section.
It prepares for __native_read_cr3() and native_write_cr3() to be used in the C entry code for handling KPTI.
--- arch/x86/include/asm/special_insns.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
arch/x86/include/asm/special_insns.h +2 -2
@@ -42,14 +42,14 @@ static __always_inline void native_write_cr2(unsigned long val) asmvolatile ("mov %0,%%cr2" : :"r" (val) :"memory" ); } -static inline unsigned long __native_read_cr3 (void ) +static __always_inline unsigned long __native_read_cr3 (void ) {unsigned long val; asm volatile("mov %%cr3,%0\n\t" : "=r" (val) : __FORCE_ORDER);return val; } -static inline void native_write_cr3 (unsigned long val) +static __always_inline void native_write_cr3(unsigned long val) { asmvolatile ("mov %0,%%cr3" : :"r" (val) :"memory" ); }
Signature · 1 line
2.19.1.6.gb485710b
PATCH 4/7 x86/entry: Add arch/x86/entry/entry64.c for C entry code
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Add a C file "entry64.c" to deposit C entry code for traps and faults which will be as the same logic as the existing ASM code in entry_64.S.
The file is as low level as entry_64.S and its code can be running in the environments that the GS base is a user controlled value, or the CR3 is the KPTI user CR3 or both.
All the code in this file should not be instrumentable. Many instrument facilities can be disabled by per-function attributes which are included in __noinstr_section. But stack-protector can not be disabled function- granularly by some compliers. So stack-protector is disabled for the whole file in Makefile.
--- arch/x86/entry/Makefile | 3 ++- arch/x86/entry/entry64.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 arch/x86/entry/entry64.c
arch/x86/entry/Makefile +2 -1
@@ -10,13 +10,14 @@ KCOV_INSTRUMENT := n CFLAGS_REMOVE_common.o = $(CC_FLAGS_FTRACE) CFLAGS_common.o += -fno-stack-protector +CFLAGS_entry64.o += -fno-stack-protector obj-y := entry_$(BITS).o thunk_$(BITS).o syscall_$(BITS).o obj-y += common.o +obj-$(CONFIG_X86_64) += entry64.o obj-y += vdso/ obj-y += vsyscall/ obj-$(CONFIG_IA32_EMULATION) += entry_64_compat.o syscall_32.o obj-$(CONFIG_X86_X32_ABI) += syscall_x32.o -
arch/x86/entry/entry64.c new file +14 -0
@@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* +* Copyright (C) 1991, 1992 Linus Torvalds +* Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs +* Copyright (C) 2000 Pavel Machek <[email protected]> +* Copyright (C) 2022 Lai Jiangshan, Ant Group +* +* Handle entries and exits for hardware traps and faults. +* +* It is as low level as entry_64.S and its code can be running in the +* environments that the GS base is a user controlled value, or the CR3 +* is the PTI user CR3 or both. +*/ +#include <asm/traps.h>
Signature · 1 line
2.19.1.6.gb485710b
PATCH 5/7 x86/entry: Add the C verion of SWITCH_TO_KERNEL_CR3 as switch_to_kernel_cr3()
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Add the C version switch_to_kernel_cr3() which implements the macro SWITCH_TO_KERNEL_CR3() in arch/x86/entry/calling.h.
No functional difference intended.
--- arch/x86/entry/entry64.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+)
arch/x86/entry/entry64.c +24 -0
@@ -12,3 +12,27 @@* is the PTI user CR3 or both. */ #include <asm/traps.h> + +#ifdef CONFIG_PAGE_TABLE_ISOLATION +static __always_inline void pti_switch_to_kernel_cr3(unsigned long user_cr3) +{ +/* +* Clear PCID and "PAGE_TABLE_ISOLATION bit", point CR3 +* at kernel pagetables: +*/ +unsigned long cr3 = user_cr3 & ~PTI_USER_PGTABLE_AND_PCID_MASK ; + +if (static_cpu_has (X86_FEATURE_PCID )) + cr3 |=X86_CR3_PCID_NOFLUSH ; + +native_write_cr3 (cr3); +} + +static __always_inline void switch_to_kernel_cr3(void) +{ +if (static_cpu_has (X86_FEATURE_PTI )) +pti_switch_to_kernel_cr3 (__native_read_cr3 ()); +} +#else +static __always_inline void switch_to_kernel_cr3(void) {} +#endif
Signature · 1 line
2.19.1.6.gb485710b
PATCH 6/7 x86/traps: Add fence_swapgs_{user,kernel}_entry() and user_entry_swapgs_and_fence()
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Add the C version fence_swapgs_{user,kernel}_entry() in entry64.c which are the same as the ASM macro FENCE_SWAPGS_{USER,KERNEL}_ENTRY.
fence_swapgs_user_entry is used in the user entry swapgs code path, to prevent a speculative swapgs when coming from kernel space.
fence_swapgs_kernel_entry is used in the kernel entry code path, to prevent the swapgs from getting speculatively skipped when coming from user space.
Add the C user_entry_swapgs_and_fence() which implements the ASM code:
swapgs
FENCE_SWAPGS_USER_ENTRY
It will be used in the user entry swapgs code path, doing the swapgs and lfence to prevent a speculative swapgs when coming from kernel space.
--- arch/x86/entry/entry64.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+)
arch/x86/entry/entry64.c +30 -0
@@ -36,3 +36,33 @@ static __always_inline void switch_to_kernel_cr3(void)#else static __always_inline void switch_to_kernel_cr3(void) {}#endif + +/* +* Mitigate Spectre v1 for conditional swapgs code paths. +* +* fence_swapgs_user_entry is used in the user entry swapgs code path, to +* prevent a speculative swapgs when coming from kernel space. It must be +* used with switch_to_kernel_cr3() in the same path. +* +* fence_swapgs_kernel_entry is used in the kernel entry code path without +* CR3 write or with conditinal CR3 write only, to prevent the swapgs from +* getting speculatively skipped when coming from user space. +* +* user_entry_swapgs_and_fence is a wrapper of swapgs and fence for user entry +* code path. +*/ +static __always_inline void fence_swapgs_user_entry(void) +{ +alternative ("" ,"lfence" ,X86_FEATURE_FENCE_SWAPGS_USER ); +} + +static __always_inline void fence_swapgs_kernel_entry(void) +{ +alternative ("" ,"lfence" ,X86_FEATURE_FENCE_SWAPGS_KERNEL ); +} + +static __always_inline void user_entry_swapgs_and_fence(void) +{ +native_swapgs (); +fence_swapgs_user_entry (); +}
Signature · 1 line
2.19.1.6.gb485710b
PATCH 7/7 x86/entry: Implement the whole error_entry() as C code
In reply to Lai JiangshanFrom: Lai Jiangshan <[email protected]>
Implement error_entry() as C code in arch/x86/entry/entry64.c and replace the ASM version of error_entry().
The code might be in the user CR3 and user GS base at the start of the function so it calls __always_inline C function only until the GS and CR3 is switched.
No functional change intended and comments are also copied.
The C version generally has better readability and easier to be updated/improved.
--- arch/x86/entry/entry64.c | 69 +++++++++++++++++++++++++++++ arch/x86/entry/entry_64.S | 85 ++---------------------------------- arch/x86/include/asm/proto.h | 1 + arch/x86/include/asm/traps.h | 1 + arch/x86/kernel/traps.c | 2 - 5 files changed, 74 insertions(+), 84 deletions(-)
arch/x86/entry/entry64.c +69 -0
@@ -13,6 +13,8 @@*/ #include <asm/traps.h> +extern unsigned char asm_load_gs_index_gs_change[]; +#ifdef CONFIG_PAGE_TABLE_ISOLATION static __always_inline void pti_switch_to_kernel_cr3(unsigned long user_cr3) {
@@ -66,3 +68,70 @@ static __always_inline void user_entry_swapgs_and_fence(void)native_swapgs ();fence_swapgs_user_entry (); } + +/* +* Put pt_regs onto the task stack and switch GS and CR3 if needed. +* The actual stack switch is done in entry_64.S. +* +* Be careful, it might be in the user CR3 and user GS base at the start +* of the function. +*/ +asmlinkage __visible __entry_text +struct pt_regs *error_entry (struct pt_regs *eregs) +{ +unsigned long iret_ip = (unsigned long )native_irq_return_iret; + +if (user_mode (eregs)) { +/* +* We entered from user mode. +* Switch to kernel gsbase and CR3. +*/ +user_entry_swapgs_and_fence (); +switch_to_kernel_cr3 (); + +/* Put pt_regs onto the task stack. */ +return sync_regs (eregs); + } + +/* +* There are two places in the kernel that can potentially fault with +* usergs. Handle them here. B stepping K8s sometimes report a +* truncated RIP for IRET exceptions returning to compat mode. Check +* for these here too. +*/ +if ((eregs->ip == iret_ip) || (eregs->ip == (unsigned int )iret_ip)) { + eregs->ip = iret_ip;/* Fix truncated RIP */ + +/* +* We came from an IRET to user mode, so we have user +* gsbase and CR3. Switch to kernel gsbase and CR3: +*/ +user_entry_swapgs_and_fence (); +switch_to_kernel_cr3 (); + +/* +* Pretend that the exception came from user mode: set up +* pt_regs as if we faulted immediately after IRET and then +* put pt_regs onto the real task stack. +*/ +return sync_regs (fixup_bad_iret (eregs)); + } + +/* +* Hack: asm_load_gs_index_gs_change can fail with user gsbase. +* If this happens, fix up gsbase and proceed. We'll fix up the +* exception and land in asm_load_gs_index_gs_change's error +* handler with kernel gsbase. +*/ +if (eregs->ip == (unsigned long )asm_load_gs_index_gs_change) +native_swapgs (); + +/* +* Issue an LFENCE to prevent GS speculation, regardless of whether +* it is a kernel or user gsbase. +*/ +fence_swapgs_kernel_entry (); + +/* Enter from kernel, don't move pt_regs */ +return eregs; +}
arch/x86/entry/entry_64.S +3 -82
109 lines not on this page · raw message
arch/x86/include/asm/proto.h +1 -0
@@ -13,6 +13,7 @@ void syscall_init(void);#ifdef CONFIG_X86_64 void entry_SYSCALL_64 (void );void entry_SYSCALL_64_safe_stack (void ); +extern unsigned char native_irq_return_iret[];long do_arch_prctl_64 (struct task_struct *task,int option,unsigned long arg2);#endif
arch/x86/include/asm/traps.h +1 -0
@@ -14,6 +14,7 @@ asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs); asmlinkage __visible notracestruct pt_regs *fixup_bad_iret (struct pt_regs *bad_regs); +asmlinkage __visible notrace struct pt_regs *error_entry(struct pt_regs *eregs);void __init trap_init (void ); asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);#endif
arch/x86/kernel/traps.c +0 -2
@@ -436,8 +436,6 @@ DEFINE_IDTENTRY_DF(exc_double_fault)#endif #ifdef CONFIG_X86_ESPFIX64 -extern unsigned char native_irq_return_iret[]; -/* * If IRET takes a non-IST fault on the espfix64 stack, then we * end up promoting it to a doublefault. In that case, take
Signature · 1 line
2.19.1.6.gb485710b