Control-flow Enforcement: Branch Tracking, PTRACE
Subsystems (12)
v8 cover Control-flow Enforcement: Branch Tracking, PTRACE
The previous version of CET Branch Tracking/PTRACE patches is here:
https://lkml.org/lkml/2019/6/6/1030
Summary of changes from v7:
Change legacy bitmap to a special mapping (patch #14). Rebase to v5.3-rc4.
Small fixes in response to comments.
H.J. Lu (5):
x86/cet/ibt: Add arch_prctl functions for IBT x86/vdso: Insert endbr32/endbr64 to vDSO x86/vdso/32: Add ENDBR32 to __kernel_vsyscall entry point x86/vsyscall/64: Add ENDBR64 to vsyscall entry points
x86: Discard .note.gnu.property sections
Yu-cheng Yu (9):
x86/cet/ibt: Add Kconfig option for user-mode Indirect Branch Tracking x86/cet/ibt: User-mode indirect branch tracking support x86/cet/ibt: Handle signals for end branch
mm/mmap: Add IBT bitmap size to address space limit check x86/cet/ibt: ELF header parsing for IBT
x86/cet/ibt: Add ENDBR to op-code-map
x86/vsyscall/64: Fixup shadow stack and branch tracking for vsyscall x86/cet: Add PTRACE interface for CET
Introduce arch_prctl(ARCH_X86_CET_MARK_LEGACY_CODE)
arch/x86/Kconfig | 17 ++ arch/x86/Makefile | 7 + arch/x86/entry/vdso/Makefile | 12 +- arch/x86/entry/vdso/vdso-layout.lds.S | 1 + arch/x86/entry/vdso/vdso32/system_call.S | 3 + arch/x86/entry/vsyscall/vsyscall_64.c | 29 +++ arch/x86/entry/vsyscall/vsyscall_emu_64.S | 9 + arch/x86/entry/vsyscall/vsyscall_trace.h | 1 + arch/x86/include/asm/cet.h | 9 + arch/x86/include/asm/disabled-features.h | 8 +- arch/x86/include/asm/fpu/regset.h | 7 +- arch/x86/include/asm/mmu_context.h | 10 + arch/x86/include/asm/processor.h | 13 +- arch/x86/include/uapi/asm/prctl.h | 3 + arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/cet.c | 54 +++++ arch/x86/kernel/cet_bitmap.c | 210 ++++++++++++++++++ arch/x86/kernel/cet_prctl.c | 19 ++ arch/x86/kernel/cpu/common.c | 17 ++ arch/x86/kernel/fpu/regset.c | 41 ++++ arch/x86/kernel/process_64.c | 5 + arch/x86/kernel/ptrace.c | 16 ++ arch/x86/kernel/vmlinux.lds.S | 10 + arch/x86/lib/x86-opcode-map.txt | 13 +- include/uapi/linux/elf.h | 1 + mm/memory.c | 8 + mm/mmap.c | 19 +- .../arch/x86/include/asm/disabled-features.h | 8 +- tools/objtool/arch/x86/lib/x86-opcode-map.txt | 13 +- 29 files changed, 552 insertions(+), 13 deletions(-) create mode 100644 arch/x86/kernel/cet_bitmap.c
Signature · 1 line
2.17.1
PATCH v8 1/14 x86/cet/ibt: Add Kconfig option for user-mode Indirect Branch Tracking
The user-mode indirect branch tracking support is done mostly by GCC to insert ENDBR64/ENDBR32 instructions at branch targets. The kernel provides CPUID enumeration and feature setup.
--- arch/x86/Kconfig | 16 ++++++++++++++++ arch/x86/Makefile | 7 +++++++ 2 files changed, 23 insertions(+)
arch/x86/Kconfig +16 -0
@@ -1940,6 +1940,9 @@ config X86_INTEL_CET config ARCH_HAS_SHSTK def_bool n +config ARCH_HAS_AS_LIMIT + def_bool n + config X86_INTEL_SHADOW_STACK_USER prompt "Intel Shadow Stack for user-mode" def_bool n
@@ -1961,6 +1964,19 @@ config X86_INTEL_SHADOW_STACK_USER If unsure, say y. +config X86_INTEL_BRANCH_TRACKING_USER + prompt "Intel Indirect Branch Tracking for user-mode" + def_bool n + depends on CPU_SUP_INTEL && X86_64 + select X86_INTEL_CET + select ARCH_HAS_AS_LIMIT + select ARCH_USE_GNU_PROPERTY + ---help--- + Indirect Branch Tracking provides hardware protection against return-/jmp- + oriented programming attacks. + + If unsure, say y + config EFI bool "EFI runtime service support" depends on ACPI
arch/x86/Makefile +7 -0
@@ -155,6 +155,13 @@ ifdef CONFIG_X86_INTEL_SHADOW_STACK_USER endif endif +# Check compiler ibt support +ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER + ifeq ($(call cc-option-yn, -fcf-protection=branch), n) + $(error CONFIG_X86_INTEL_BRANCH_TRACKING_USER not supported by compiler) + endif +endif + # # If the function graph tracer is used with mcount instead of fentry, # '-maccumulate-outgoing-args' is needed to prevent a GCC bug
Signature · 1 line
2.17.1
PATCH v8 2/14 x86/cet/ibt: User-mode indirect branch tracking support
In reply to Yu-cheng YuAdd user-mode indirect branch tracking enabling/disabling and supporting routines.
--- arch/x86/include/asm/cet.h | 6 ++++ arch/x86/include/asm/disabled-features.h | 8 ++++- arch/x86/kernel/cet.c | 36 +++++++++++++++++++ arch/x86/kernel/cpu/common.c | 17 +++++++++ .../arch/x86/include/asm/disabled-features.h | 8 ++++- 5 files changed, 73 insertions(+), 2 deletions(-)
arch/x86/include/asm/cet.h +6 -0
@@ -16,6 +16,8 @@ struct cet_status {unsigned long shstk_size; unsigned int locked:1; unsigned int shstk_enabled:1; + unsigned int ibt_enabled:1; + unsigned int ibt_bitmap_used:1; };#ifdef CONFIG_X86_INTEL_CET
@@ -27,6 +29,8 @@ void cet_disable_shstk(void);void cet_disable_free_shstk (struct task_struct *p);int cet_restore_signal (bool ia32,struct sc_ext *sc);int cet_setup_signal (bool ia32,unsigned long rstor,struct sc_ext *sc); +int cet_setup_ibt (void ); +void cet_disable_ibt (void );#else static inline int prctl_cet (int option,unsigned long arg2) {return -EINVAL ; }static inline int cet_setup_shstk (void ) {return -EINVAL ; }
@@ -37,6 +41,8 @@ static inline void cet_disable_free_shstk(struct task_struct *p) {}static inline int cet_restore_signal (bool ia32,struct sc_ext *sc) {return -EINVAL ; }static inline int cet_setup_signal (bool ia32,unsigned long rstor,struct sc_ext *sc) {return -EINVAL ; } +static inline int cet_setup_ibt (void ) {return -EINVAL ; } +static inline void cet_disable_ibt (void ) {}#endif #define cpu_x86_cet_enabled () \
arch/x86/include/asm/disabled-features.h +7 -1
@@ -68,6 +68,12 @@#define DISABLE_SHSTK (1<<(X86_FEATURE_SHSTK & 31))#endif +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER +#define DISABLE_IBT 0 +#else +#define DISABLE_IBT (1<<(X86_FEATURE_IBT & 31)) +#endif +/* * Make sure to add features to the correct mask */
@@ -89,7 +95,7 @@#define DISABLED_MASK15 0#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP|DISABLE_SHSTK)#define DISABLED_MASK17 0 -#define DISABLED_MASK18 0 +#define DISABLED_MASK18 (DISABLE_IBT)#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)#endif /* _ASM_X86_DISABLED_FEATURES_H */
arch/x86/kernel/cet.c +36 -0
@@ -13,6 +13,8 @@#include <linux/uaccess.h> #include <linux/sched/signal.h> #include <linux/compat.h> +#include <linux/vmalloc.h> +#include <linux/bitops.h> #include <asm/msr.h> #include <asm/user.h> #include <asm/fpu/internal.h>
@@ -325,3 +327,37 @@ int cet_setup_signal(bool ia32, unsigned long rstor_addr, struct sc_ext *sc_ext)modify_fpu_regs_end ();return 0 ; } + +int cet_setup_ibt (void ) +{ +u64 r; + +if (!cpu_feature_enabled (X86_FEATURE_IBT )) +return -EOPNOTSUPP ; + +modify_fpu_regs_begin (); +rdmsrl (MSR_IA32_U_CET , r); + r |= (MSR_IA32_CET_ENDBR_EN |MSR_IA32_CET_NO_TRACK_EN ); +wrmsrl (MSR_IA32_U_CET , r); +modify_fpu_regs_end (); + + current->thread.cet.ibt_enabled =1 ; +return 0 ; +} + +void cet_disable_ibt (void ) +{ +u64 r; + +if (!cpu_feature_enabled (X86_FEATURE_IBT )) +return ; + +modify_fpu_regs_begin (); +rdmsrl (MSR_IA32_U_CET , r); + r &= ~(MSR_IA32_CET_ENDBR_EN |MSR_IA32_CET_LEG_IW_EN | +MSR_IA32_CET_NO_TRACK_EN |MSR_IA32_CET_BITMAP_MASK ); +wrmsrl (MSR_IA32_U_CET , r); +modify_fpu_regs_end (); + + current->thread.cet.ibt_enabled =0 ; +}
arch/x86/kernel/cpu/common.c +17 -0
@@ -512,6 +512,23 @@ static __init int setup_disable_shstk(char *s)__setup ("no_cet_shstk" , setup_disable_shstk);#endif +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER +static __init int setup_disable_ibt (char *s) +{ +/* require an exact match without trailing characters */ +if (s[0 ] !='\0' ) +return 0 ; + +if (!boot_cpu_has (X86_FEATURE_IBT )) +return 1 ; + +setup_clear_cpu_cap (X86_FEATURE_IBT ); +pr_info ("x86: 'no_cet_ibt' specified, disabling Branch Tracking\n" ); +return 1 ; +} +__setup ("no_cet_ibt" , setup_disable_ibt); +#endif +/* * Some CPU features depend on higher CPUID levels, which may not always * be available due to CPUID level capping or broken virtualization
tools/arch/x86/include/asm/disabled-features.h +7 -1
@@ -68,6 +68,12 @@#define DISABLE_SHSTK (1<<(X86_FEATURE_SHSTK & 31))#endif +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER +#define DISABLE_IBT 0 +#else +#define DISABLE_IBT (1<<(X86_FEATURE_IBT & 31)) +#endif +/* * Make sure to add features to the correct mask */
@@ -89,7 +95,7 @@#define DISABLED_MASK15 0#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP|DISABLE_SHSTK)#define DISABLED_MASK17 0 -#define DISABLED_MASK18 0 +#define DISABLED_MASK18 (DISABLE_IBT)#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)#endif /* _ASM_X86_DISABLED_FEATURES_H */
Signature · 1 line
2.17.1
PATCH v8 3/14 x86/cet/ibt: Handle signals for end branch
In reply to Yu-cheng YuRestore end branch tracking setting from thread header.
--- arch/x86/kernel/cet.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+)
arch/x86/kernel/cet.c +18 -0
@@ -281,6 +281,15 @@ int cet_restore_signal(bool ia32, struct sc_ext *sc_ext) msr_ia32_u_cet |=MSR_IA32_CET_SHSTK_EN ; } +if (current->thread.cet.ibt_enabled) { +if (current->thread.cet.ibt_bitmap_used) + msr_ia32_u_cet |= (IBT_BITMAP_ADDR | +MSR_IA32_CET_LEG_IW_EN ); + + msr_ia32_u_cet |= (MSR_IA32_CET_ENDBR_EN | +MSR_IA32_CET_NO_TRACK_EN ); + } +wrmsrl (MSR_IA32_PL3_SSP , new_ssp);wrmsrl (MSR_IA32_U_CET , msr_ia32_u_cet);return 0 ;
@@ -321,6 +330,15 @@ int cet_setup_signal(bool ia32, unsigned long rstor_addr, struct sc_ext *sc_ext) sc_ext->ssp = new_ssp; } +if (current->thread.cet.ibt_enabled) { +if (current->thread.cet.ibt_bitmap_used) + msr_ia32_u_cet |= (IBT_BITMAP_ADDR | +MSR_IA32_CET_LEG_IW_EN ); + + msr_ia32_u_cet |= (MSR_IA32_CET_ENDBR_EN | +MSR_IA32_CET_NO_TRACK_EN ); + } +modify_fpu_regs_begin ();wrmsrl (MSR_IA32_PL3_SSP , ssp);wrmsrl (MSR_IA32_U_CET , msr_ia32_u_cet);
Signature · 1 line
2.17.1
PATCH v8 4/14 mm/mmap: Add IBT bitmap size to address space limit check
In reply to Yu-cheng YuThe indirect branch tracking legacy bitmap takes a large address space. This causes may_expand_vm() failure on the address limit check. For a IBT-enabled task, add the bitmap size to the address limit.
--- arch/x86/include/asm/mmu_context.h | 10 ++++++++++ mm/mmap.c | 19 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-)
arch/x86/include/asm/mmu_context.h +10 -0
@@ -360,6 +360,16 @@ static inline unsigned long __get_current_cr3_fast(void)return cr3; } +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER +static inline unsigned long arch_as_limit (void ) +{ +if (current->thread.cet.ibt_enabled) +return IBT_BITMAP_SIZE ; +else +return 0 ; +} +#endif +typedef struct {struct mm_struct *mm; }temp_mm_state_t ;
mm/mmap.c +18 -1
@@ -3283,13 +3283,30 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,return NULL ; } +#ifndef CONFIG_ARCH_HAS_AS_LIMIT +static inline unsigned long arch_as_limit (void ) +{ +return 0 ; +} +#endif +/* * Return true if the calling process may expand its vm space by the passed * number of pages */ bool may_expand_vm (struct mm_struct *mm,vm_flags_t flags,unsigned long npages) { -if (mm->total_vm + npages >rlimit (RLIMIT_AS ) >>PAGE_SHIFT ) +unsigned long as_limit =rlimit (RLIMIT_AS ); +unsigned long as_limit_plus = as_limit +arch_as_limit (); + +/* as_limit_plus overflowed */ +if (as_limit_plus < as_limit) + as_limit_plus =RLIM_INFINITY ; + +if (as_limit_plus > as_limit) + as_limit = as_limit_plus; + +if (mm->total_vm + npages > as_limit >>PAGE_SHIFT )return false; if (is_data_mapping(flags) &&
Signature · 1 line
2.17.1
PATCH v8 5/14 x86/cet/ibt: ELF header parsing for IBT
In reply to Yu-cheng YuLook in .note.gnu.property of an ELF file and check if Indirect Branch Tracking needs to be enabled for the task.
--- arch/x86/Kconfig | 1 + arch/x86/kernel/process_64.c | 5 +++++ 2 files changed, 6 insertions(+)
arch/x86/Kconfig +1 -0
@@ -1971,6 +1971,7 @@ config X86_INTEL_BRANCH_TRACKING_USER select X86_INTEL_CET select ARCH_HAS_AS_LIMIT select ARCH_USE_GNU_PROPERTY + select ARCH_BINFMT_ELF_STATE ---help--- Indirect Branch Tracking provides hardware protection against return-/jmp- oriented programming attacks.
arch/x86/kernel/process_64.c +5 -0
@@ -856,6 +856,11 @@ int arch_setup_property(struct arch_elf_state *state) return r; } +if (cpu_feature_enabled (X86_FEATURE_IBT )) { +if (state->gnu_property &GNU_PROPERTY_X86_FEATURE_1_IBT ) + r =cet_setup_ibt (); + } +return r; }#endif
Signature · 1 line
2.17.1
PATCH v8 6/14 x86/cet/ibt: Add arch_prctl functions for IBT
In reply to Yu-cheng YuFrom: "H.J. Lu" <[email protected]>
Update ARCH_X86_CET_STATUS and ARCH_X86_CET_DISABLE to include Indirect Branch Tracking features.
--- arch/x86/include/uapi/asm/prctl.h | 2 ++ arch/x86/kernel/cet_prctl.c | 4 ++++ 2 files changed, 6 insertions(+)
arch/x86/include/uapi/asm/prctl.h +2 -0
@@ -18,5 +18,7 @@#define ARCH_X86_CET_DISABLE 0x3002#define ARCH_X86_CET_LOCK 0x3003#define ARCH_X86_CET_ALLOC_SHSTK 0x3004 +#define ARCH_X86_CET_GET_LEGACY_BITMAP 0x3005/* deprecated */ +#define ARCH_X86_CET_SET_LEGACY_BITMAP 0x3006/* deprecated */ #endif /* _ASM_X86_PRCTL_H */
arch/x86/kernel/cet_prctl.c +4 -0
@@ -20,6 +20,8 @@ static int handle_get_status(unsigned long arg2)if (current->thread.cet.shstk_enabled) features |=GNU_PROPERTY_X86_FEATURE_1_SHSTK ; +if (current->thread.cet.ibt_enabled) + features |=GNU_PROPERTY_X86_FEATURE_1_IBT ; shstk_base = current->thread.cet.shstk_base; shstk_size = current->thread.cet.shstk_size;
@@ -69,6 +71,8 @@ int prctl_cet(int option, unsigned long arg2)return -EPERM ;if (arg2 &GNU_PROPERTY_X86_FEATURE_1_SHSTK )cet_disable_free_shstk (current); +if (arg2 &GNU_PROPERTY_X86_FEATURE_1_IBT ) +cet_disable_ibt ();return 0 ;
Signature · 1 line
2.17.1
PATCH v8 7/14 x86/cet/ibt: Add ENDBR to op-code-map
In reply to Yu-cheng YuAdd control transfer terminating instructions:
ENDBR64/ENDBR32:
Mark a valid 64/32-bit control transfer endpoint.
--- arch/x86/lib/x86-opcode-map.txt | 13 +++++++++++-- tools/objtool/arch/x86/lib/x86-opcode-map.txt | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-)
arch/x86/lib/x86-opcode-map.txt +11 -2
@@ -620,7 +620,16 @@ ea: SAVEPREVSSP (f3) # Skip 0xeb-0xff EndTable -Table: 3-byte opcode 2 (0x0f 0x38) +Table: 3-byte opcode 2 (0x0f 0x1e) +Referrer: +AVXcode: +# Skip 0x00-0xf9 +fa: ENDBR64 (f3) +fb: ENDBR32 (f3) +#skip 0xfc-0xff +EndTable + +Table: 3-byte opcode 3 (0x0f 0x38) Referrer: 3-byte escape 1 AVXcode: 2 # 0x0f 0x38 0x00-0x0f
@@ -804,7 +813,7 @@ f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSS Pq,Qq f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) EndTable -Table: 3-byte opcode 3 (0x0f 0x3a) +Table: 3-byte opcode 4 (0x0f 0x3a) Referrer: 3-byte escape 2 AVXcode: 3 # 0x0f 0x3a 0x00-0xff
tools/objtool/arch/x86/lib/x86-opcode-map.txt +11 -2
@@ -620,7 +620,16 @@ ea: SAVEPREVSSP (f3) # Skip 0xeb-0xff EndTable -Table: 3-byte opcode 2 (0x0f 0x38) +Table: 3-byte opcode 2 (0x0f 0x1e) +Referrer: +AVXcode: +# Skip 0x00-0xf9 +fa: ENDBR64 (f3) +fb: ENDBR32 (f3) +#skip 0xfc-0xff +EndTable + +Table: 3-byte opcode 3 (0x0f 0x38) Referrer: 3-byte escape 1 AVXcode: 2 # 0x0f 0x38 0x00-0x0f
@@ -804,7 +813,7 @@ f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSS Pq,Qq f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) EndTable -Table: 3-byte opcode 3 (0x0f 0x3a) +Table: 3-byte opcode 4 (0x0f 0x3a) Referrer: 3-byte escape 2 AVXcode: 3 # 0x0f 0x3a 0x00-0xff
Signature · 1 line
2.17.1
Re: [PATCH v8 07/14] x86/cet/ibt: Add ENDBR to op-code-map
Quoted 16 lines · Yu-cheng Yu
Add control transfer terminating instructions:
ENDBR64/ENDBR32:
Mark a valid 64/32-bit control transfer endpoint.Signed-off-by Yu-cheng Yu <[email protected]>--- arch/x86/lib/x86-opcode-map.txt | 13 +++++++++++-- tools/objtool/arch/x86/lib/x86-opcode-map.txt | 13 +++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-)arch/x86/lib/x86-opcode-map.txt +0
-0
arch/x86/lib/x86-opcode-map.txt ea: SAVEPREVSSP (f3) · line 628
Quoted 10 lines
arch/x86/lib/x86-opcode-map.txt +6
-1@@ -620,7 +620,16 @@ ea: SAVEPREVSSP (f3) # Skip 0xeb-0xff EndTable -Table: 3-byte opcode 2 (0x0f 0x38) +Table: 3-byte opcode 2 (0x0f 0x1e) +Referrer: +AVXcode: +# Skip 0x00-0xf9 +fa: ENDBR64 (f3) +fb: ENDBR32 (f3)
endbr32 and endbr64 have 2-byte opcodes (0F 1E) and a ModRM byte, so a new Grp is needed
Quoted 48 lines
arch/x86/lib/x86-opcode-map.txt +5
-1+#skip 0xfc-0xff +EndTable + +Table: 3-byte opcode 3 (0x0f 0x38) Referrer: 3-byte escape 1 AVXcode: 2 # 0x0f 0x38 0x00-0x0f@@ -804,7 +813,7 @@ f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSS Pq,Qq f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) EndTable -Table: 3-byte opcode 3 (0x0f 0x3a) +Table: 3-byte opcode 4 (0x0f 0x3a) Referrer: 3-byte escape 2 AVXcode: 3 # 0x0f 0x3a 0x00-0xfftools/objtool/arch/x86/lib/x86-opcode-map.txt +11
-2@@ -620,7 +620,16 @@ ea: SAVEPREVSSP (f3) # Skip 0xeb-0xff EndTable -Table: 3-byte opcode 2 (0x0f 0x38) +Table: 3-byte opcode 2 (0x0f 0x1e) +Referrer: +AVXcode: +# Skip 0x00-0xf9 +fa: ENDBR64 (f3) +fb: ENDBR32 (f3) +#skip 0xfc-0xff +EndTable + +Table: 3-byte opcode 3 (0x0f 0x38) Referrer: 3-byte escape 1 AVXcode: 2 # 0x0f 0x38 0x00-0x0f@@ -804,7 +813,7 @@ f6: ADCX Gy,Ey (66) | ADOX Gy,Ey (F3) | MULX By,Gy,rDX,Ey (F2),(v) | WRSS Pq,Qq f7: BEXTR Gy,Ey,By (v) | SHLX Gy,Ey,By (66),(v) | SARX Gy,Ey,By (F3),(v) | SHRX Gy,Ey,By (F2),(v) EndTable -Table: 3-byte opcode 3 (0x0f 0x3a) +Table: 3-byte opcode 4 (0x0f 0x3a) Referrer: 3-byte escape 2 AVXcode: 3 # 0x0f 0x3a 0x00-0xff
PATCH v8 8/14 x86/vdso: Insert endbr32/endbr64 to vDSO
In reply to Yu-cheng YuFrom: "H.J. Lu" <[email protected]>
When Intel indirect branch tracking is enabled, functions in vDSO which may be called indirectly must have endbr32 or endbr64 as the first instruction. Compiler must support -fcf-protection=branch so that it can be used to compile vDSO.
--- arch/x86/entry/vdso/Makefile | 12 +++++++++++- arch/x86/entry/vdso/vdso-layout.lds.S | 1 + 2 files changed, 12 insertions(+), 1 deletion(-)
arch/x86/entry/vdso/Makefile +11 -1
@@ -114,13 +114,17 @@ vobjx32s := $(foreach F,$(vobjx32s-y),$(obj)/$F) # Convert 64bit object file to x32 for x32 vDSO. quiet_cmd_x32 = X32 $@ - cmd_x32 = $(OBJCOPY) -O elf32-x86-64 $< $@ + cmd_x32 = $(OBJCOPY) -R .note.gnu.property -O elf32-x86-64 $< $@ $(obj)/%-x32.o: $(obj)/%.o FORCE $(call if_changed,x32) targets += vdsox32.lds $(vobjx32s-y) +ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER + $(obj)/vclock_gettime.o $(obj)/vgetcpu.o $(obj)/vdso32/vclock_gettime.o: KBUILD_CFLAGS += -fcf-protection=branch +endif + $(obj)/%.so: OBJCOPYFLAGS := -S $(obj)/%.so: $(obj)/%.so.dbg FORCE $(call if_changed,objcopy)
@@ -178,6 +182,12 @@ quiet_cmd_vdso = VDSO $@ VDSO_LDFLAGS = -shared --hash-style=both --build-id \ $(call ld-option, --eh-frame-hdr) -Bsymbolic +ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER + VDSO_LDFLAGS += $(call ldoption, -z$(comma)ibt) +endif +ifdef CONFIG_X86_INTEL_SHADOW_STACK_USER + VDSO_LDFLAGS += $(call ldoption, -z$(comma)shstk) +endif GCOV_PROFILE := n quiet_cmd_vdso_and_check = VDSO $@
arch/x86/entry/vdso/vdso-layout.lds.S +1 -0
@@ -52,6 +52,7 @@ SECTIONS *(.gnu.linkonce.b.*) } :text + .note.gnu.property : { *(.note.gnu.property) } :text :note .note : { *(.note.*) } :text :note .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr
Signature · 1 line
2.17.1
PATCH v8 9/14 x86/vdso/32: Add ENDBR32 to __kernel_vsyscall entry point
In reply to Yu-cheng YuFrom: "H.J. Lu" <[email protected]>
Add ENDBR32 to __kernel_vsyscall entry point.
--- arch/x86/entry/vdso/vdso32/system_call.S | 3 +++ 1 file changed, 3 insertions(+)
arch/x86/entry/vdso/vdso32/system_call.S +3 -0
@@ -14,6 +14,9 @@ ALIGN __kernel_vsyscall: CFI_STARTPROC +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER + endbr32 +#endif /* * Reshuffle regs so that all of any of the entry instructions * will preserve enough state.
Signature · 1 line
2.17.1
PATCH v8 10/14 x86/vsyscall/64: Add ENDBR64 to vsyscall entry points
In reply to Yu-cheng YuFrom: "H.J. Lu" <[email protected]>
Add ENDBR64 to vsyscall entry points.
--- arch/x86/entry/vsyscall/vsyscall_emu_64.S | 9 +++++++++ 1 file changed, 9 insertions(+)
arch/x86/entry/vsyscall/vsyscall_emu_64.S +9 -0
@@ -17,16 +17,25 @@ __PAGE_ALIGNED_DATA .type __vsyscall_page, @object __vsyscall_page: +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER + endbr64 +#endif mov $__NR_gettimeofday, %rax syscall ret .balign 1024, 0xcc +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER + endbr64 +#endif mov $__NR_time, %rax syscall ret .balign 1024, 0xcc +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER + endbr64 +#endif mov $__NR_getcpu, %rax syscall ret
Signature · 1 line
2.17.1
PATCH v8 11/14 x86/vsyscall/64: Fixup shadow stack and branch tracking for vsyscall
In reply to Yu-cheng YuWhen emulating a RET, also unwind the task's shadow stack and cancel the current branch tracking status.
--- arch/x86/entry/vsyscall/vsyscall_64.c | 29 ++++++++++++++++++++++++ arch/x86/entry/vsyscall/vsyscall_trace.h | 1 + 2 files changed, 30 insertions(+)
arch/x86/entry/vsyscall/vsyscall_64.c +29 -0
@@ -38,6 +38,9 @@#include <asm/fixmap.h> #include <asm/traps.h> #include <asm/paravirt.h> +#include <asm/fpu/xstate.h> +#include <asm/fpu/types.h> +#include <asm/fpu/internal.h> #define CREATE_TRACE_POINTS #include "vsyscall_trace.h"
@@ -286,6 +289,32 @@ bool emulate_vsyscall(unsigned long error_code,/* Emulate a ret instruction. */ regs->ip = caller; regs->sp +=8 ; + +/* Unwind shadow stack. */ + +#ifdef CONFIG_X86_INTEL_SHADOW_STACK_USER +if (current->thread.cet.shstk_enabled) { +u64 r; + +modify_fpu_regs_begin (); +rdmsrl (MSR_IA32_PL3_SSP , r); +wrmsrl (MSR_IA32_PL3_SSP , r +8 ); +modify_fpu_regs_end (); + } +#endif + +/* Fixup branch tracking */ +#ifdef CONFIG_X86_INTEL_BRANCH_TRACKING_USER +if (current->thread.cet.ibt_enabled) { +u64 r; + +modify_fpu_regs_begin (); +rdmsrl (MSR_IA32_U_CET , r); +wrmsrl (MSR_IA32_U_CET , r & ~MSR_IA32_CET_WAIT_ENDBR ); +modify_fpu_regs_end (); + } +#endif +return true; sigsegv:
arch/x86/entry/vsyscall/vsyscall_trace.h +1 -0
@@ -25,6 +25,7 @@ TRACE_EVENT(emulate_vsyscall,#endif #undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE#define TRACE_INCLUDE_PATH ../../arch/x86/entry/vsyscall/ #define TRACE_INCLUDE_FILE vsyscall_trace#include <trace/define_trace.h>
Signature · 1 line
2.17.1
PATCH v8 12/14 x86/cet: Add PTRACE interface for CET
In reply to Yu-cheng YuAdd REGSET_CET64/REGSET_CET32 to get/set CET MSRs:
IA32_U_CET (user-mode CET settings) and
IA32_PL3_SSP (user-mode shadow stack)
--- arch/x86/include/asm/fpu/regset.h | 7 +++--- arch/x86/kernel/fpu/regset.c | 41 +++++++++++++++++++++++++++++++ arch/x86/kernel/ptrace.c | 16 ++++++++++++ include/uapi/linux/elf.h | 1 + 4 files changed, 62 insertions(+), 3 deletions(-)
arch/x86/include/asm/fpu/regset.h +4 -3
@@ -7,11 +7,12 @@#include <linux/regset.h> -extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active; +extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active, + cetregs_active;extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, - xstateregs_get; + xstateregs_get, cetregs_get;extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, - xstateregs_set; + xstateregs_set, cetregs_set;/* * xstateregs_active == regset_fpregs_active. Please refer to the comment
arch/x86/kernel/fpu/regset.c +41 -0
47 lines not on this page · raw message
arch/x86/kernel/ptrace.c +16 -0
@@ -52,7 +52,9 @@ enum x86_regset {REGSET_IOPERM64 =REGSET_XFP ,REGSET_XSTATE ,REGSET_TLS , +REGSET_CET64 =REGSET_TLS ,REGSET_IOPERM32 , + REGSET_CET32, }; struct pt_regs_offset {
@@ -1239,6 +1241,13 @@ static struct user_regset x86_64_regsets[] __ro_after_init = { .size = sizeof(long), .align = sizeof(long), .active = ioperm_active, .get = ioperm_get }, + [REGSET_CET64] = { + .core_note_type = NT_X86_CET, + .n = sizeof(struct cet_user_state) / sizeof(u64), + .size = sizeof(u64), .align = sizeof(u64), + .active = cetregs_active, .get = cetregs_get, + .set = cetregs_set + }, };static const struct user_regset_view user_x86_64_view = {
@@ -1294,6 +1303,13 @@ static struct user_regset x86_32_regsets[] __ro_after_init = { .size = sizeof(u32), .align = sizeof(u32), .active = ioperm_active, .get = ioperm_get }, + [REGSET_CET32] = { + .core_note_type = NT_X86_CET, + .n = sizeof(struct cet_user_state) / sizeof(u64), + .size = sizeof(u64), .align = sizeof(u64), + .active = cetregs_active, .get = cetregs_get, + .set = cetregs_set + }, };static const struct user_regset_view user_x86_32_view = {
include/uapi/linux/elf.h +1 -0
@@ -401,6 +401,7 @@ typedef struct elf64_shdr {#define NT_386_TLS 0x200/* i386 TLS slots (struct user_desc) */ #define NT_386_IOPERM 0x201/* x86 io permission bitmap (1=deny) */ #define NT_X86_XSTATE 0x202/* x86 extended state using xsave */ +#define NT_X86_CET 0x203/* x86 cet state */ #define NT_S390_HIGH_GPRS 0x300/* s390 upper register halves */ #define NT_S390_TIMER 0x301/* s390 timer register */ #define NT_S390_TODCMP 0x302/* s390 TOD clock comparator register */
Signature · 1 line
2.17.1
PATCH v8 13/14 x86: Discard .note.gnu.property sections
In reply to Yu-cheng YuFrom: "H.J. Lu" <[email protected]>
With the command-line option, -mx86-used-note=yes, the x86 assembler in binutils 2.32 and above generates a program property note in a note section, .note.gnu.property, to encode used x86 ISAs and features. To exclude .note.gnu.property sections from NOTE segment in x86 kernel linker script:
PHDRS {
text PT_LOAD FLAGS(5);
data PT_LOAD FLAGS(6);
percpu PT_LOAD FLAGS(6);
init PT_LOAD FLAGS(7);
note PT_NOTE FLAGS(0);
}
SECTIONS
{
...
.notes : AT(ADDR(.notes) - 0xffffffff80000000) { __start_notes = .; KEEP(*(.not
e.*)) __stop_notes = .; } :text :note
...
}
this patch discards .note.gnu.property sections in kernel linker script by adding
/DISCARD/ : {
*(.note.gnu.property)
}
before .notes sections. Since .exit.text and .exit.data sections are discarded at runtime, it undefines EXIT_TEXT and EXIT_DATA to exclude .exit.text and .exit.data sections from default discarded sections.
--- arch/x86/kernel/vmlinux.lds.S | 10 ++++++++++ 1 file changed, 10 insertions(+)
arch/x86/kernel/vmlinux.lds.S +10 -0
@@ -146,6 +146,10 @@ SECTIONS _etext = .; } :text = 0x9090 + /* .note.gnu.property sections should be discarded */ + /DISCARD/ : { + *(.note.gnu.property) + } NOTES :text :note EXCEPTION_TABLE(16) :text = 0x9090
@@ -415,6 +419,12 @@ SECTIONS STABS_DEBUG DWARF_DEBUG + /* Sections to be discarded. EXIT_TEXT and EXIT_DATA discard at + * runtime, not link time. */ +#undef EXIT_TEXT +#define EXIT_TEXT +#undef EXIT_DATA +#define EXIT_DATA DISCARDS /DISCARD/ : { *(.eh_frame)
Signature · 1 line
2.17.1
PATCH v8 14/14 Introduce arch_prctl(ARCH_X86_CET_MARK_LEGACY_CODE)
In reply to Yu-cheng YuWhen CET Indirect Branch Tracking (IBT) is enabled, the processor expects every branch target is an ENDBR instruction, or the target's address is marked as legacy in the legacy code bitmap. The bitmap covers the whole user-mode address space (TASK_SIZE_MAX for 64-bit, TASK_SIZE for IA32), and each bit represents one page of linear address range. The bitmap is located at the topmost address: (TASK_SIZE - IBT_BITMAP_SIZE).
It is allocated only when the first time ARCH_X86_MARK_LEGACY_CODE is called from an application.
The IBT bitmap is visiable from user-mode, but not writable.
Introduce:
arch_prctl(ARCH_X86_CET_MARK_LEGACY_CODE, unsigned long *buf)
Mark an address range as IBT legacy code.
*buf: starting linear address
*(buf + 1): size of the legacy code
*(buf + 2): set (1); clear (0)
--- arch/x86/include/asm/cet.h | 3 + arch/x86/include/asm/processor.h | 13 +- arch/x86/include/uapi/asm/prctl.h | 1 + arch/x86/kernel/Makefile | 2 +- arch/x86/kernel/cet_bitmap.c | 210 ++++++++++++++++++++++++++++++ arch/x86/kernel/cet_prctl.c | 15 +++ mm/memory.c | 8 ++ 7 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 arch/x86/kernel/cet_bitmap.c
arch/x86/include/asm/cet.h +3 -0
@@ -4,6 +4,7 @@#ifndef __ASSEMBLY__#include <linux/types.h> +#include <asm/processor.h> struct task_struct ;struct sc_ext ;
@@ -30,6 +31,7 @@ void cet_disable_free_shstk(struct task_struct *p);int cet_restore_signal (bool ia32,struct sc_ext *sc);int cet_setup_signal (bool ia32,unsigned long rstor,struct sc_ext *sc);int cet_setup_ibt (void ); +int cet_mark_legacy_code (unsigned long addr,unsigned long size,unsigned long set);void cet_disable_ibt (void );#else static inline int prctl_cet (int option,unsigned long arg2) {return -EINVAL ; }
@@ -42,6 +44,7 @@ static inline int cet_restore_signal(bool ia32, struct sc_ext *sc) { return -EINstatic inline int cet_setup_signal (bool ia32,unsigned long rstor,struct sc_ext *sc) {return -EINVAL ; }static inline int cet_setup_ibt (void ) {return -EINVAL ; } +static inline int cet_mark_legacy_code (unsigned long addr,unsigned long size,unsigned long set) {return -EINVAL ; }static inline void cet_disable_ibt (void ) {}#endif
arch/x86/include/asm/processor.h +12 -1
@@ -888,7 +888,18 @@ static inline void spin_lock_prefetch(const void *x)#define TASK_SIZE_OF (child) ((test_tsk_thread_flag(child, TIF_ADDR32)) ? \ IA32_PAGE_OFFSET : TASK_SIZE_MAX) -#define STACK_TOP TASK_SIZE_LOW +#define MMAP_MAX (unsigned long)(test_thread_flag(TIF_ADDR32) ? \ + TASK_SIZE : TASK_SIZE_MAX) + +#define IBT_BITMAP_SIZE (round_up(MMAP_MAX, PAGE_SIZE * BITS_PER_BYTE) / \ + (PAGE_SIZE * BITS_PER_BYTE)) + +#define IBT_BITMAP_ADDR (TASK_SIZE - IBT_BITMAP_SIZE) + +#define STACK_TOP (TASK_SIZE_LOW < IBT_BITMAP_ADDR - PAGE_SIZE ? \ + TASK_SIZE_LOW : \ + IBT_BITMAP_ADDR - PAGE_SIZE) +#define STACK_TOP_MAX TASK_SIZE_MAX#define INIT_THREAD { \
arch/x86/include/uapi/asm/prctl.h +1 -0
@@ -20,5 +20,6 @@#define ARCH_X86_CET_ALLOC_SHSTK 0x3004#define ARCH_X86_CET_GET_LEGACY_BITMAP 0x3005/* deprecated */ #define ARCH_X86_CET_SET_LEGACY_BITMAP 0x3006/* deprecated */ +#define ARCH_X86_CET_MARK_LEGACY_CODE 0x3007#endif /* _ASM_X86_PRCTL_H */
arch/x86/kernel/Makefile +1 -1
@@ -140,7 +140,7 @@ obj-$(CONFIG_UNWINDER_ORC) += unwind_orc.o obj-$(CONFIG_UNWINDER_FRAME_POINTER) += unwind_frame.o obj-$(CONFIG_UNWINDER_GUESS) += unwind_guess.o -obj-$(CONFIG_X86_INTEL_CET) += cet.o cet_prctl.o +obj-$(CONFIG_X86_INTEL_CET) += cet.o cet_prctl.o cet_bitmap.o ### # 64 bit specific files
arch/x86/kernel/cet_bitmap.c new file +210 -0
210 lines not on this page · raw message
arch/x86/kernel/cet_prctl.c +15 -0
@@ -57,6 +57,18 @@ static int handle_alloc_shstk(unsigned long arg2)return 0 ; } +static int handle_mark_legacy_code (unsigned long arg2) +{ +unsigned long addr, size, set; + +if (get_user (addr, (unsigned long __user *)arg2) || +get_user (size, (unsigned long __user *)arg2 +1 ) || +get_user (set, (unsigned long __user *)arg2 +2 )) +return -EFAULT ; + +return cet_mark_legacy_code (addr, size, set); +} +int prctl_cet (int option,unsigned long arg2) {if (!cpu_x86_cet_enabled ())
@@ -83,6 +95,9 @@ int prctl_cet(int option, unsigned long arg2)case ARCH_X86_CET_ALLOC_SHSTK :return handle_alloc_shstk (arg2); +case ARCH_X86_CET_MARK_LEGACY_CODE : +return handle_mark_legacy_code (arg2); +default :return -EINVAL ; }
mm/memory.c +8 -0
@@ -3290,6 +3290,12 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,flush_icache_page (vma, page); entry =mk_pte (page, vma->vm_page_prot); + +if (is_zero_pfn (pte_pfn (entry))) { + entry =pte_mkspecial (entry); + goto alloc_set_pte_out; + } +if (write) entry =maybe_mkwrite (pte_mkdirty (entry), vma);/* copy-on-write page */
@@ -3302,6 +3308,8 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,inc_mm_counter_fast (vma->vm_mm,mm_counter_file (page));page_add_file_rmap (page, false); } + +alloc_set_pte_out:set_pte_at (vma->vm_mm, vmf->address, vmf->pte, entry);/* no need to invalidate: a not-present page won't be cached */
Signature · 1 line
2.17.1